diff --git a/.asf.yaml b/.asf.yaml index 509defa46414a5..00614ab35019a6 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -193,9 +193,6 @@ github: - Yukang-Lian - Hastyshell - yujun777 - - shuke987 - - wm1581066 - - doris-robot - ixzc - feiniaofeiafei - wyxxxcat diff --git a/.github/scripts/emit_litefuse_otel_io.py b/.github/scripts/emit_litefuse_otel_io.py index d11e465df87484..4ef45f088797de 100644 --- a/.github/scripts/emit_litefuse_otel_io.py +++ b/.github/scripts/emit_litefuse_otel_io.py @@ -1513,7 +1513,7 @@ def parse_args(): parser.add_argument("--pr-number", default="") parser.add_argument("--head-sha", default="") parser.add_argument("--base-sha", default="") - parser.add_argument("--model", default="gpt-5.5") + parser.add_argument("--model", default="gpt-5.6-sol") parser.add_argument("--reasoning-effort", default="") parser.add_argument("--environment", default="github-actions") parser.add_argument("--max-input-chars", type=int, default=200_000) diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index 030d6463e56443..df911dc7ce5da4 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -19,6 +19,7 @@ name: BE UT (macOS) on: pull_request: + types: [opened, synchronize] schedule: - cron: '0 4,10,16,22 * * *' # schedule it periodically to share the cache @@ -56,7 +57,7 @@ jobs: max-size: "5G" restore-keys: BE-UT-macOS- - - name: Run UT ${{ github.ref }} + - name: Build BE ${{ github.ref }} if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }} run: | cellars=( @@ -80,21 +81,50 @@ jobs: 'openjdk@11' 'maven' 'node' - 'llvm@16' + 'llvm@20' + 'libomp' ) brew install "${cellars[@]}" || true + # macos-15 runners are Apple Silicon (arm64), so download the arm64 + # prebuilt thirdparty. Using the x86_64 archive makes the linker ignore + # every thirdparty static library ("found architecture 'x86_64', + # required architecture 'arm64'") and the final BE link fails with + # thousands of undefined symbols. pushd thirdparty branch="${{ github.base_ref }}" if [[ -z "${branch}" ]] || [[ "${branch}" == 'master' ]]; then - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-darwin-x86_64.tar.xz \ - -o doris-thirdparty-prebuilt-darwin-x86_64.tar.xz + curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-darwin-arm64.tar.xz \ + -o doris-thirdparty-prebuilt-darwin-arm64.tar.xz else - curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-${branch/branch-/}/doris-thirdparty-prebuilt-darwin-x86_64.tar.xz" \ - -o doris-thirdparty-prebuilt-darwin-x86_64.tar.xz + curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-${branch/branch-/}/doris-thirdparty-prebuilt-darwin-arm64.tar.xz" \ + -o doris-thirdparty-prebuilt-darwin-arm64.tar.xz fi - tar -xvf doris-thirdparty-prebuilt-darwin-x86_64.tar.xz + tar -xvf doris-thirdparty-prebuilt-darwin-arm64.tar.xz popd - export JAVA_HOME="${JAVA_HOME_17_X64%\/}" - ./run-be-ut.sh --run -j "$(nproc)" --clean + # macos-15 runners are Apple Silicon (arm64), so the JDK env var is + # JAVA_HOME_17_arm64. Fall back to the x64 variable for Intel runners. + JAVA_HOME="${JAVA_HOME_17_arm64:-${JAVA_HOME_17_X64}}" + export JAVA_HOME="${JAVA_HOME%\/}" + + # AppleClang ships no OpenMP runtime, so CMake's find_package(OpenMP) + # (e.g. in contrib/openblas) fails with "Could NOT find OpenMP_C" and + # "'omp.h' file not found". Point the compiler/linker at Homebrew's + # libomp. be/CMakeLists.txt resets CMAKE_C_FLAGS/CMAKE_CXX_FLAGS, so + # EXTRA_CXX_FLAGS is also needed to pass the include path to the BE build. + LIBOMP_PREFIX="$(brew --prefix libomp)" + export CPPFLAGS="-I${LIBOMP_PREFIX}/include ${CPPFLAGS:-}" + export CFLAGS="-I${LIBOMP_PREFIX}/include ${CFLAGS:-}" + export CXXFLAGS="-I${LIBOMP_PREFIX}/include ${CXXFLAGS:-}" + export LDFLAGS="-L${LIBOMP_PREFIX}/lib ${LDFLAGS:-}" + export EXTRA_CXX_FLAGS="-I${LIBOMP_PREFIX}/include ${EXTRA_CXX_FLAGS:-}" + + # Only verify that the BE compiles on macOS; do NOT compile be/test. + # The unit tests are built and run locally, not in this job. build.sh + # configures the BE with -DMAKE_TEST=OFF, so nothing under be/test is + # compiled. Skip the Java extensions and cdc client (not relevant to a + # macOS C++ build check), and pass -j explicitly because build.sh + # otherwise defaults to only ~nproc/4 jobs. + DISABLE_BE_JAVA_EXTENSIONS=ON DISABLE_BE_CDC_CLIENT=ON \ + ./build.sh --be -j "$(nproc)" diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index 4814480a5dfed8..17ebad717f01cc 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -21,7 +21,6 @@ name: Code Formatter on: pull_request: - pull_request_target: workflow_dispatch: issue_comment: types: [ created ] @@ -35,24 +34,17 @@ jobs: name: "Clang Formatter" runs-on: ubuntu-latest if: | - (github.event_name == 'pull_request') || (github.event_name == 'pull_request_target') || + (github.event_name == 'pull_request') || (github.event_name == 'issue_comment' && github.event.comment.body == 'run buildall' && github.actor == 'doris-robot' && github.event.issue.user.login == 'github-actions[bot]') steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - if: ${{ github.event_name != 'pull_request_target' }} - uses: actions/checkout@v3 + uses: actions/checkout@v7 with: persist-credentials: false - - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} ) - if: ${{ github.event_name == 'pull_request_target' }} - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Checkout paths-filter run: | rm -rf ./.github/actions/paths-filter diff --git a/.github/workflows/code-review-runner.yml b/.github/workflows/code-review-runner.yml index 2eaa302d05441c..433ffaf01af8f4 100644 --- a/.github/workflows/code-review-runner.yml +++ b/.github/workflows/code-review-runner.yml @@ -46,6 +46,7 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ inputs.head_sha }} + fetch-depth: 0 - name: Install ripgrep run: | @@ -104,7 +105,13 @@ jobs: install -m 700 -d "$RUNNER_TEMP/codex-home" printf 'CODEX_HOME=%s\n' "$RUNNER_TEMP/codex-home" >> "$GITHUB_ENV" - ossutil -i "$OSS_AK" -k "$OSS_SK" -e "$OSS_ENDPOINT" cp -f "$OSS_CODEX_AUTH_OBJECT" "$RUNNER_TEMP/codex-home/auth.json" + auth_object="$(printf '%s\n' \ + 'oss://doris-community-ci/codex/auth.json.1' \ + 'oss://doris-community-ci/codex/auth.json.2' \ + | shuf -n 1)" + printf 'CODEX_AUTH_OSS_OBJECT=%s\n' "$auth_object" >> "$GITHUB_ENV" + echo "Selected Codex auth object: ${auth_object##*/}" + ossutil -i "$OSS_AK" -k "$OSS_SK" -e "$OSS_ENDPOINT" cp -f "$auth_object" "$RUNNER_TEMP/codex-home/auth.json" chmod 600 "$RUNNER_TEMP/codex-home/auth.json" test -s "$RUNNER_TEMP/codex-home/auth.json" jq -e ' @@ -138,7 +145,6 @@ jobs: OSS_AK: ${{ secrets.OSS_AK }} OSS_SK: ${{ secrets.OSS_SK }} OSS_ENDPOINT: oss-cn-hongkong.aliyuncs.com - OSS_CODEX_AUTH_OBJECT: oss://doris-community-ci/codex/auth.json - name: Sync Codex memories from OSS run: | @@ -251,6 +257,7 @@ jobs: fi - name: Prepare authoritative PR context and required AGENTS guides + id: review_context env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} @@ -265,35 +272,6 @@ jobs: exit 1 fi - retry_to_file() { - local output_file="$1" - shift - local attempt delay=2 tmp_file - tmp_file="$(mktemp "${output_file}.tmp.XXXXXX")" - - for attempt in 1 2 3 4; do - if "$@" > "$tmp_file"; then - mv "$tmp_file" "$output_file" - return 0 - fi - - if [ "$attempt" -lt 4 ]; then - echo "Attempt ${attempt}/4 failed while preparing ${output_file}; retrying in ${delay}s..." - sleep "$delay" - delay=$((delay * 2)) - fi - done - - rm -f "$tmp_file" - return 1 - } - - retry_to_file "$REVIEW_CONTEXT_DIR/pr_changed_files.txt" \ - gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files?per_page=100" \ - --jq '.[].filename' - retry_to_file "$REVIEW_CONTEXT_DIR/pr.diff" \ - gh pr diff "$PR_NUMBER" --repo "$REPO" --color=never - live_pr="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}")" live_head_sha="$(jq -r '.head.sha' <<<"$live_pr")" live_base_sha="$(jq -r '.base.sha' <<<"$live_pr")" @@ -304,6 +282,25 @@ jobs: exit 1 fi + if [ "$(git rev-parse --is-shallow-repository)" != "false" ]; then + echo "Full history is required to generate an authoritative three-dot PR diff" + exit 1 + fi + + diff_range="${BASE_SHA}...${HEAD_SHA}" + merge_base_sha="$(git merge-base "$BASE_SHA" "$HEAD_SHA")" + echo "Generating review context from $diff_range (merge base: $merge_base_sha)" + git rev-list --count "${merge_base_sha}..${HEAD_SHA}" + + changed_files_tmp="$(mktemp "$REVIEW_CONTEXT_DIR/pr_changed_files.txt.tmp.XXXXXX")" + git diff --name-only --no-ext-diff "$diff_range" > "$changed_files_tmp" + mv "$changed_files_tmp" "$REVIEW_CONTEXT_DIR/pr_changed_files.txt" + + diff_tmp="$(mktemp "$REVIEW_CONTEXT_DIR/pr.diff.tmp.XXXXXX")" + git diff --no-ext-diff --no-color "$diff_range" > "$diff_tmp" + mv "$diff_tmp" "$REVIEW_CONTEXT_DIR/pr.diff" + git diff --shortstat "$diff_range" + helper="$RUNNER_TEMP/prepare_review_agents.py" gh api \ -H "Accept: application/vnd.github.raw" \ @@ -340,11 +337,12 @@ jobs: - User review focus: PLACEHOLDER_CONTEXT_DIR/review_focus.txt - PR changed files: PLACEHOLDER_CONTEXT_DIR/pr_changed_files.txt - Authoritative aggregate PR diff: PLACEHOLDER_CONTEXT_DIR/pr.diff + - PR diff range: merge-base(PLACEHOLDER_BASE_SHA, PLACEHOLDER_HEAD_SHA) to PLACEHOLDER_HEAD_SHA - Required AGENTS.md files: PLACEHOLDER_CONTEXT_DIR/required_agents.txt - Shared subagent review ledger: PLACEHOLDER_CONTEXT_DIR/subagent_review_findings.md PR diff and path-reading (for all subagents and the main agent): - - PLACEHOLDER_CONTEXT_DIR/pr.diff is the authoritative aggregate diff for the whole PR, and PLACEHOLDER_CONTEXT_DIR/pr_changed_files.txt is the authoritative changed-path list. Do not obtain the PR diff or changed-path list through other means. + - PLACEHOLDER_CONTEXT_DIR/pr.diff and PLACEHOLDER_CONTEXT_DIR/pr_changed_files.txt were both generated locally with `git diff PLACEHOLDER_BASE_SHA...PLACEHOLDER_HEAD_SHA` after the live PR base/head were verified. They are the authoritative PR diff and changed-path list: they contain changes from the computed merge base to the PR head. The listed PR Base SHA identifies the target-branch snapshot and is not necessarily the diff's left endpoint. Do not obtain the PR diff or changed-path list through other means. - Before reading any file whose exact path is not already confirmed by PLACEHOLDER_CONTEXT_DIR/pr_changed_files.txt, PLACEHOLDER_CONTEXT_DIR/pr.diff, or a previous successful command output, you MUST first run `rg --files` to confirm the actual path of the target file. Before reviewing any code, you MUST read and follow the code review skill in this repository. During review, you must strictly follow those instructions. @@ -507,7 +505,7 @@ jobs: # Codex uses bubblewrap for that mode and uid maps can be unavailable. codex exec --goal "$GOAL_PROMPT" \ --cd "$GITHUB_WORKSPACE" \ - --model "gpt-5.5" \ + --model "gpt-5.6-sol" \ --config "model_reasoning_effort=xhigh" \ --sandbox danger-full-access \ --color never \ @@ -608,7 +606,7 @@ jobs: --pr-number "$PR_NUMBER" \ --head-sha "$HEAD_SHA" \ --base-sha "$BASE_SHA" \ - --model "gpt-5.5" \ + --model "gpt-5.6-sol" \ --reasoning-effort "xhigh" \ --environment "github-actions" \ --max-payload-bytes 4000000 \ @@ -650,6 +648,11 @@ jobs: - name: Sync Codex auth back to OSS if: ${{ always() }} run: | + if [ -z "$CODEX_AUTH_OSS_OBJECT" ]; then + echo "No selected Codex auth object found; skipping OSS auth sync." + exit 0 + fi + if [ ! -s "$CODEX_HOME/auth.json" ]; then echo "No Codex auth file found; skipping OSS auth sync." exit 0 @@ -660,22 +663,26 @@ jobs: and (.tokens.access_token | type == "string" and length > 0) and (.tokens.refresh_token | type == "string" and length > 0) ' "$CODEX_HOME/auth.json" >/dev/null - ossutil -i "$OSS_AK" -k "$OSS_SK" -e "$OSS_ENDPOINT" cp -f "$CODEX_HOME/auth.json" "$OSS_CODEX_AUTH_OBJECT" + ossutil -i "$OSS_AK" -k "$OSS_SK" -e "$OSS_ENDPOINT" cp -f "$CODEX_HOME/auth.json" "$CODEX_AUTH_OSS_OBJECT" env: OSS_AK: ${{ secrets.OSS_AK }} OSS_SK: ${{ secrets.OSS_SK }} OSS_ENDPOINT: oss-cn-hongkong.aliyuncs.com - OSS_CODEX_AUTH_OBJECT: oss://doris-community-ci/codex/auth.json - name: Comment PR on review failure - if: ${{ always() && steps.review.outcome != 'success' }} + if: ${{ always() && (steps.review_context.outcome != 'success' || steps.review.outcome != 'success') }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REVIEW_CONTEXT_OUTCOME: ${{ steps.review_context.outcome }} REVIEW_FAILURE_REASON: ${{ steps.review.outputs.failure_reason }} REVIEW_OUTCOME: ${{ steps.review.outcome }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | - error_msg="${REVIEW_FAILURE_REASON:-Review step was $REVIEW_OUTCOME (possibly timeout or cancelled)}" + if [ "$REVIEW_CONTEXT_OUTCOME" != "success" ]; then + error_msg="Review context preparation failed before Codex ran; inspect the 'Prepare authoritative PR context and required AGENTS guides' step." + else + error_msg="${REVIEW_FAILURE_REASON:-Review step was $REVIEW_OUTCOME (possibly timeout or cancelled)}" + fi gh pr comment "${{ inputs.pr_number }}" --body "$(cat <- - github.event_name == 'pull_request_target' && + github.event_name == 'pull_request' && steps.changed-files.outputs.added_modified != '' env: CHANGED_FILES: ${{ steps.changed-files.outputs.added_modified }} @@ -110,7 +108,7 @@ jobs: - name: Check License if: >- - github.event_name != 'pull_request_target' || + github.event_name != 'pull_request' || steps.changed-files.outputs.config_file != '' uses: apache/skywalking-eyes@v0.8.0 env: diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp index efb2930f8364fb..582db25ff6d667 100644 --- a/be/benchmark/benchmark_main.cpp +++ b/be/benchmark/benchmark_main.cpp @@ -17,6 +17,11 @@ #include +#include +#include +#include +#include + #include "benchmark_arrow_validation.hpp" #include "benchmark_bit_pack.hpp" #include "benchmark_bits.hpp" @@ -29,14 +34,18 @@ #include "benchmark_fmod.hpp" #include "benchmark_hll_merge.hpp" #include "benchmark_hybrid_set.hpp" +#include "benchmark_pdep_unpack.hpp" #include "benchmark_string.hpp" #include "benchmark_string_replace.hpp" #include "benchmark_zone_map_index.hpp" #include "binary_cast_benchmark.hpp" +#include "common/config.h" #include "core/block/block.h" #include "core/column/column_string.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_string.h" +#include "parquet/benchmark_parquet_decoder.hpp" +#include "parquet/benchmark_parquet_reader.hpp" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/memory/thread_mem_tracker_mgr.h" @@ -50,6 +59,35 @@ namespace doris { // change if need +static bool init_benchmark_config(const char* executable) { + std::vector candidates; + if (const char* doris_home = std::getenv("DORIS_HOME"); doris_home != nullptr) { + candidates.emplace_back(std::filesystem::path(doris_home) / "conf" / "be.conf"); + } + candidates.emplace_back(std::filesystem::current_path() / "conf" / "be.conf"); + const auto executable_dir = std::filesystem::absolute(executable).parent_path(); + candidates.emplace_back(executable_dir / ".." / ".." / "conf" / "be.conf"); + candidates.emplace_back(executable_dir / ".." / ".." / ".." / "conf" / "be.conf"); + for (const auto& candidate : candidates) { + if (!std::filesystem::exists(candidate)) { + continue; + } + if (std::getenv("DORIS_HOME") == nullptr) { + const auto inferred_home = candidate.parent_path().parent_path().string(); + // be.conf contains paths relative to DORIS_HOME. Keep standalone benchmark launches + // equivalent to build/test scripts after locating the same repository config. + if (::setenv("DORIS_HOME", inferred_home.c_str(), 0) != 0) { + return false; + } + } + // Mutable config storage is populated by config::init. Reader benchmarks must use the + // production defaults because zero-initialized safety limits reject every valid footer. + return config::init(candidate.c_str(), false); + } + std::cerr << "Unable to find conf/be.conf for benchmark initialization\n"; + return false; +} + static void Example1(benchmark::State& state) { // init. dont time it. state.PauseTiming(); @@ -76,6 +114,11 @@ BENCHMARK(Example1); // ThreadContext + mem tracker, otherwise the allocator throws E-7412. Mirrors // the minimal subset of be/test/testutil/run_all_tests.cpp::main. int main(int argc, char** argv) { + if (!doris::init_benchmark_config(argv[0])) { + return 1; + } + doris::config::enable_bmi2_optimizations = true; + SCOPED_INIT_THREAD_CONTEXT(); doris::ExecEnv::GetInstance()->init_mem_tracker(); doris::thread_context()->thread_mem_tracker_mgr->init(); diff --git a/be/benchmark/benchmark_pdep_unpack.hpp b/be/benchmark/benchmark_pdep_unpack.hpp new file mode 100644 index 00000000000000..6589947591745f --- /dev/null +++ b/be/benchmark/benchmark_pdep_unpack.hpp @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) + +#include + +#include +#include +#include + +#include "util/bit_stream_utils.inline.h" +#include "util/pdep_unpack.h" + +namespace doris { +namespace { + +constexpr int kPdepUnpackBatchSize = 32; +constexpr int kPdepUnpackL1NumValues = 1 << 12; +constexpr int kPdepUnpackL2NumValues = 1 << 18; +constexpr int kPdepUnpackLargeNumValues = 1 << 20; +const std::vector kPdepUnpackNumValues = { + 32, 64, 128, kPdepUnpackL1NumValues, kPdepUnpackL2NumValues, kPdepUnpackLargeNumValues}; + +template +void scalar_unpack(const uint8_t* input, int num_values, T* output) { + constexpr int bytes_per_batch = kPdepUnpackBatchSize * BIT_WIDTH / 8; + for (int i = 0; i < num_values; i += kPdepUnpackBatchSize) { + BitPacking::Unpack32Values(input, bytes_per_batch, output + i); + input += bytes_per_batch; + } +} + +template +void pdep_unpack(const uint8_t* input, int num_values, T* output) { + for (int i = 0; i < num_values; i += kPdepUnpackBatchSize) { + PdepUnpack::unpack32(input, output + i); + input += kPdepUnpackBatchSize * BIT_WIDTH / 8; + } +} + +template +void unpack_if_supported(const uint8_t* input, int num_values, T* output) { + if constexpr (PdepUnpack::is_supported_type()) { + if constexpr (USE_PDEP) { + pdep_unpack(input, num_values, output); + } else { + scalar_unpack(input, num_values, output); + } + } else { + __builtin_unreachable(); + } +} + +template +void unpack(int bit_width, const uint8_t* input, int num_values, T* output) { +#define UNPACK_CASE(width) \ + case width: \ + unpack_if_supported(input, num_values, output); \ + return + switch (bit_width) { + UNPACK_CASE(1); + UNPACK_CASE(2); + UNPACK_CASE(3); + UNPACK_CASE(4); + UNPACK_CASE(5); + UNPACK_CASE(6); + UNPACK_CASE(7); + UNPACK_CASE(8); + UNPACK_CASE(9); + UNPACK_CASE(10); + UNPACK_CASE(11); + UNPACK_CASE(12); + UNPACK_CASE(13); + UNPACK_CASE(14); + UNPACK_CASE(15); + UNPACK_CASE(16); + UNPACK_CASE(17); + UNPACK_CASE(18); + UNPACK_CASE(19); + UNPACK_CASE(20); + UNPACK_CASE(21); + UNPACK_CASE(22); + UNPACK_CASE(23); + UNPACK_CASE(24); + UNPACK_CASE(25); + UNPACK_CASE(26); + UNPACK_CASE(27); + UNPACK_CASE(28); + UNPACK_CASE(29); + UNPACK_CASE(30); + UNPACK_CASE(31); + UNPACK_CASE(32); + default: + __builtin_unreachable(); + } +#undef UNPACK_CASE +} + +template +struct PdepUnpackBenchmarkData { + PdepUnpackBenchmarkData(int bit_width, int num_values) + : input(num_values * bit_width / 8), + scalar_output(num_values), + pdep_output(num_values) { + std::mt19937_64 rng(0x17993); + for (auto& byte : input) { + byte = static_cast(rng()); + } + unpack(bit_width, input.data(), num_values, scalar_output.data()); + } + + std::vector input; + std::vector scalar_output; + std::vector pdep_output; +}; + +template +void BM_ScalarUnpack(benchmark::State& state) { + const int bit_width = static_cast(state.range(0)); + const int num_values = static_cast(state.range(1)); + PdepUnpackBenchmarkData data(bit_width, num_values); + for (auto _ : state) { + unpack(bit_width, data.input.data(), num_values, data.scalar_output.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * num_values); +} + +template +void BM_PdepUnpack(benchmark::State& state) { + const int bit_width = static_cast(state.range(0)); + const int num_values = static_cast(state.range(1)); + if (!PdepUnpack::is_supported()) { + state.SkipWithError("CPU does not support BMI2 and AVX2"); + return; + } + PdepUnpackBenchmarkData data(bit_width, num_values); + unpack(bit_width, data.input.data(), num_values, data.pdep_output.data()); + if (data.scalar_output != data.pdep_output) { + state.SkipWithError("PDEP+AVX2 output differs from scalar output"); + return; + } + for (auto _ : state) { + unpack(bit_width, data.input.data(), num_values, data.pdep_output.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * num_values); +} + +template +void BM_ActualUnpack(benchmark::State& state) { + const int bit_width = static_cast(state.range(0)); + const int num_values = static_cast(state.range(1)); + PdepUnpackBenchmarkData data(bit_width, num_values); + auto result = BitPacking::UnpackValues(bit_width, data.input.data(), data.input.size(), + num_values, data.pdep_output.data()); + if (data.scalar_output != data.pdep_output) { + state.SkipWithError("Actual output differs from scalar output"); + return; + } + for (auto _ : state) { + result = BitPacking::UnpackValues(bit_width, data.input.data(), data.input.size(), + num_values, data.pdep_output.data()); + benchmark::DoNotOptimize(result); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * num_values); +} + +#define REGISTER_PDEP_UNPACK_BENCHMARK(type, max_bit_width) \ + BENCHMARK_TEMPLATE(BM_ScalarUnpack, type) \ + ->ArgsProduct( \ + {benchmark::CreateDenseRange(1, max_bit_width, 1), kPdepUnpackNumValues}); \ + BENCHMARK_TEMPLATE(BM_PdepUnpack, type) \ + ->ArgsProduct( \ + {benchmark::CreateDenseRange(1, max_bit_width, 1), kPdepUnpackNumValues}); \ + BENCHMARK_TEMPLATE(BM_ActualUnpack, type) \ + ->ArgsProduct( \ + {benchmark::CreateDenseRange(1, max_bit_width, 1), kPdepUnpackNumValues}) + +REGISTER_PDEP_UNPACK_BENCHMARK(uint8_t, 8); +REGISTER_PDEP_UNPACK_BENCHMARK(uint16_t, 16); +REGISTER_PDEP_UNPACK_BENCHMARK(uint32_t, 32); + +#undef REGISTER_PDEP_UNPACK_BENCHMARK + +} // namespace +} // namespace doris + +#endif diff --git a/be/benchmark/parquet/AGENTS.md b/be/benchmark/parquet/AGENTS.md new file mode 100644 index 00000000000000..a8667fb43cdf5f --- /dev/null +++ b/be/benchmark/parquet/AGENTS.md @@ -0,0 +1,296 @@ +# Parquet microbenchmark guide for agents + +This file applies to `be/benchmark/parquet/`. Read it before changing, running, or interpreting +these benchmarks. The current suite is a local benchmark foundation, not the complete Parquet +benchmark system described in the design document. + +## What exists today + +The benchmark binary registers two groups: + +- `ParquetDecoder`: native page decoder benchmarks using in-memory encoded pages. +- `ParquetReader`: local-file benchmarks that call the format V2 Parquet reader directly. + +The relevant files are: + +- `benchmark_parquet_decoder.hpp`: deterministic page construction and decoder registration. +- `benchmark_parquet_reader.hpp`: deterministic local Parquet fixtures and reader registration. +- `parquet_benchmark_scenarios.h`: scenario definitions and the selected matrix. +- `README.md`: short human-oriented build and invocation examples. +- `be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp`: matrix invariants. + +Do not describe this suite as end-to-end SQL, `FileScannerV2`, remote I/O, V1/V2 comparison, or a +cross-engine benchmark. The reader benchmark starts at `format::parquet::ParquetReader` and does +not include FE planning, scanner scheduling, `TableReader`, client latency, or Runtime Profile +collection. + +## Build and list cases + +Performance results must come from a Release build: + +```shell +./build.sh --benchmark -j128 +``` + +List all Parquet cases and verify the expected registration counts: + +```shell +be/output/lib/benchmark_test --benchmark_list_tests \ + | grep -E '^Parquet(Decoder|Reader)/' + +be/output/lib/benchmark_test --benchmark_list_tests \ + | grep -c '^ParquetDecoder/' # currently 152 + +be/output/lib/benchmark_test --benchmark_list_tests \ + | grep -c '^ParquetReader/' # currently 137 +``` + +When running the binary directly from `be/build_RELEASE/bin`, make sure the JVM and third-party +libraries are discoverable. Prefer the installed `be/output/lib/benchmark_test` produced by +`build.sh` because the normal Doris environment already configures its dependencies. + +## Run smoke verification + +A smoke run proves that every registered case initializes and completes. It does not produce a +stable performance baseline: + +```shell +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetDecoder/' \ + --benchmark_min_time=0.001s \ + --benchmark_out=parquet-decoder-smoke.json \ + --benchmark_out_format=json + +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetReader/' \ + --benchmark_min_time=0.001s \ + --benchmark_out=parquet-reader-smoke.json \ + --benchmark_out_format=json +``` + +Reject a smoke run if the process is non-zero, the expected number of JSON results is absent, or +any result contains `error_occurred`. Also run the scenario matrix unit test after changing the +matrix. Never use the 1 ms smoke measurements to claim a speedup or regression. + +## Run a performance comparison + +Compare the same named cases on the same host, compiler, build flags, CPU set, NUMA node, and power +policy. Use the same fixture directory and cache state for both revisions. A suitable local command +is: + +```shell +taskset -c 8 be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetReader/predicate_scan/plain/null_50/alternating/sel_10/' \ + --benchmark_min_time=1s \ + --benchmark_repetitions=10 \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=parquet-reader.json \ + --benchmark_out_format=json +``` + +Use at least three untimed warmups before collecting local warm-cache results. Run revisions in an +ABBA order when comparing two commits. Do not compare results from different CPUs, cache +topologies, compiler versions, or build types. + +The suite does not currently control the OS page cache. Repeated reader cases are normally +warm-cache measurements. Do not label them cold-NVMe results. Do not clear a shared machine's page +cache to manufacture a cold run. + +## Current scenario matrix + +`ParquetDecoder` contains 19 encoding/type pairs. Each pair is run at 1%, 10%, 50%, and 100% +selection with clustered and alternating selection ranges, for 152 registered cases. + +| Encoding | Physical types | +|---|---| +| PLAIN | INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY | +| Dictionary | INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY | +| BYTE_STREAM_SPLIT | FLOAT, DOUBLE, FIXED_LEN_BYTE_ARRAY | +| DELTA_BINARY_PACKED | INT32, INT64 | +| DELTA_LENGTH_BYTE_ARRAY | BYTE_ARRAY | +| DELTA_BYTE_ARRAY | BYTE_ARRAY | + +`ParquetReader` deliberately uses a single-variable matrix rather than a Cartesian product. After +deduplication it contains 137 cases covering: + +- operations: open-to-first-block, full scan, predicate scan, limit 1, and limit 1000; +- file encodings: PLAIN, dictionary, BYTE_STREAM_SPLIT, and DELTA_BINARY_PACKED; +- null ratios: 0%, 1%, 10%, 50%, and 90%; +- null shapes: clustered and alternating; +- predicate selectivities: 0%, 1%, 10%, 50%, 90%, and 100%; +- projection shapes: predicate-only and predicate plus one lazy payload column; +- schema widths: 4, 32, 128, and 512 columns; +- predicate position: first or last column. + +Except for the axis being varied, reader cases inherit the baseline: nullable INT32, PLAIN, +alternating 10% nulls, 10% selectivity, 32 columns, predicate at column zero, and predicate plus +payload projection. + +## How decoder data is generated + +Decoder pages are constructed in memory before the timed loop. There is no Parquet file, Python +generator, random seed, or manifest involved. + +- Every page contains 65,536 logical values. +- Integer values use `(row * 17) % 1000003`. +- Floating-point values use `(row % 1009) * 0.25 - 100.0`. +- Binary values are fixed at 16 bytes. Their first bytes contain `row % 1009` and the remaining + bytes are deterministic filler. +- PLAIN fixed-width values are copied in physical byte order. PLAIN BYTE_ARRAY values use a + four-byte length followed by payload bytes. +- BYTE_STREAM_SPLIT pages transpose the byte lanes of the fixed-width PLAIN representation. +- DELTA pages are produced with the Arrow Parquet DELTA encoders. +- Dictionary pages contain 256 deterministic entries. IDs repeat from 0 through 255 and are + encoded with an eight-bit RLE/bit-packed stream. +- Clustered selection is one continuous range. Alternating selection spreads selected rows evenly + and intentionally produces many one-row physical ranges. + +Page generation, selection construction, decoder creation, dictionary setup, and `set_data` are +outside the timed decode call. The sinks consume decoder callbacks and prevent compiler removal, +but they do not build a Doris `Column`. Consequently these cases isolate decoder traversal and +selection cost; they do not measure definition-level decoding, nullable reconstruction, type +conversion, or full column materialization. + +## How reader Parquet files are generated + +Reader fixtures are generated lazily by C++ code using Arrow builders and the Arrow Parquet writer. +They are stored under: + +```text +${TMPDIR:-/tmp}/doris_parquet_reader_benchmark/ +``` + +Fixture contents and writer settings are: + +- 16,384 rows and 4 row groups of 4,096 rows each; +- all columns are nullable INT32 and reuse the same deterministic Arrow array; +- every non-null value is `row % 100`; +- the predicate is `value < selectivity_percent`, so the threshold maps directly to the intended + non-null selectivity; +- alternating nulls use a 101-row period and `(row * 37) % 101`, avoiding direct correlation with + the 100-value predicate period; +- clustered nulls use contiguous null prefixes inside each 1,024-row cluster; +- Parquet format version 2.6, DataPage V2, no compression, and statistics disabled; +- dictionary is explicitly enabled only for dictionary fixtures; other fixtures disable dictionary + and request their named encoding; +- after writing, every column chunk in every row group is checked through footer metadata to ensure + that the writer did not silently choose another encoding. + +The fixture name depends only on axes that change file contents. Therefore predicate thresholds, +projection mode, and operation can reuse the same file. Existing fixtures are footer-validated +before reuse. Delete only the dedicated `doris_parquet_reader_benchmark` temporary directory when +fixture rules change; never remove a broad temporary or workspace directory. + +Fixture creation and footer verification happen before benchmark iterations. Reader initialization +and `close()` are excluded from steady-state full/predicate/LIMIT timing. The +`open_to_first_block` case intentionally includes reader construction, metadata loading, `open()`, +and the first `get_block()` call. + +## Interpret the result correctly + +Google Benchmark JSON contains `real_time`, `cpu_time`, repetitions, and custom counters. Use: + +- `cpu_time` for decoder CPU efficiency; +- both `real_time` and `cpu_time` for reader cases, because latency and CPU cost answer different + questions; +- median across repetitions as the primary result; +- p95 and a confidence interval for automated gates; never select the fastest repetition. + +Custom counters mean: + +- `raw_rows`: logical source rows considered by the case; +- `selected_rows`: rows returned after the predicate or selection; +- `ns/raw_row`: normalized source-row cost and the primary comparison across selectivities; +- `ns/selected_row`: cost per surviving row; it naturally rises as selectivity falls; +- `selection_ranges`: decoder physical ranges; compare alternating with clustered at the same + selectivity to expose per-range overhead; +- `fixture_bytes` or `encoded_bytes`: fixture/page size, not peak memory; +- `items_per_second`: selected rows per second, so it is not directly comparable across different + selectivities; +- `bytes_per_second`: a logical benchmark counter. In reader cases it is estimated from projected + INT32 values, not measured storage-device bytes or compressed bytes. + +Useful comparisons are: + +- clustered versus alternating at equal selectivity: sparse-range/cursor overhead; +- null 0% versus 1/10/50/90% at equal selection: definition-level and nullable reconstruction + overhead in the reader; +- predicate-only versus predicate-projected: lazy materialization benefit; +- width 4/32/128/512 with predicate first/last: schema and metadata traversal overhead; +- PLAIN versus dictionary/BYTE_STREAM_SPLIT/DELTA at the same reader baseline: encoding path cost; +- open-to-first-block and LIMIT cases separately from steady-state full scans. + +Before attributing a regression, confirm that output row counts are identical, the benchmark name +and fixture encoding match, and repetition variability is acceptable. A coefficient of variation +above 3% is inconclusive and should be rerun. Correlate CPU regressions with `perf stat` counters +such as cycles, instructions, branch misses, and cache misses when possible. + +## Coverage audit and required follow-up + +The current matrix is not comprehensive. Preserve this distinction in PR descriptions and reports. + +### P0: complete the local benchmark phase + +1. Add the design's `matrix.yaml`, deterministic corpus generator, `manifest.json`, checksum, and + standalone corpus verifier. The current runtime-generated files cannot be shared unchanged with + V1, StarRocks, or DuckDB. +2. Add correctness oracles. Benchmarks must validate consumed counts and representative checksums + outside the timed region before their performance samples are trusted. +3. Add reader-level INT64, FLOAT, DOUBLE, BYTE_ARRAY/string, FIXED_LEN_BYTE_ARRAY, DATE, + TIMESTAMP, and DECIMAL cases. Today only nullable INT32 reaches the complete reader path. +4. Extend decoder coverage with 0% and 90% selection, definition levels/null reconstruction, + dictionary conversion, and real Doris `Column` materialization. The current decoder sink does + not cover those costs or report decoded bytes per second. +5. Add the representative nullable sparse corpus requested by the design: 32 INT64 columns, 128 + row groups, and enough rows to exercise many pages. The current 16K-row/four-row-group fixture + is a smoke-sized workload. +6. Add dictionary footprint sweeps based on actual bytes relative to L1/L2 cache, value-width + sweeps, cardinality sweeps, and dictionary-to-PLAIN fallback/mixed-page files. The current + dictionary is small and fixed. +7. Add a Doris V1/V2 SQL runner over the same verified corpus, correctness comparison, Runtime + Profile collection, normalized result JSON, comparison report, and the first stable local + baseline. The current 1 ms run only proves executability. + +### P1: broaden Parquet format and execution behavior + +1. Vary compression codecs: uncompressed, Snappy, ZSTD, LZ4, Brotli, and Gzip where supported. +2. Cover DataPage V1 and V2, page size, row-group size/count, page boundaries, small files, and + large files that exceed cache capacity. +3. Add required/non-nullable columns, correlated and anti-correlated null/predicate distributions, + random deterministic runs, long null runs, and selections that cross page boundaries. +4. Add Boolean, INT96 compatibility, logical annotations, timestamp units/time zones, decimal + physical representations, short/long strings, skew, and empty values. +5. Add nested ARRAY/MAP/STRUCT and repeated definition/repetition-level cases. +6. Add row-group statistics, page index, Bloom filter, dictionary filtering, all-filtered batches, + missing-column/default-value handling, schema mapping, and schema-evolution cases. Keep pruning + benchmarks separate from decode benchmarks so skipped work is not misattributed to a faster + decoder. +7. Add predicate LIMIT 1/1000 cases. Current LIMIT cases only vary reader batch size without a + predicate. +8. Add peak tracked memory, allocation counts, and an explicit warm/cold-cache methodology. + +### Later phases + +Remote S3/FileCache/RTT workloads, prefetch metrics, cross-engine adapters, nightly/weekly jobs, and +release regression gates belong to later phases. They require isolated infrastructure and must not +be simulated by silently changing the local reader benchmark. + +## Current validation record + +At commit `16e05dd5c71`, a Release build completed and the matrix unit test passed 4/4. A 1 ms smoke +run executed 152 decoder and 137 reader cases with zero benchmark errors. This is an execution +record only. It is not a reviewed performance baseline because repetitions, host isolation, +warmups, cache control, `perf` data, variance, and before/after comparison were not collected. + +## Rules for extending the suite + +- Keep deterministic data construction and validation outside timed regions. +- Verify actual footer encodings; never trust writer configuration alone. +- Change one primary axis at a time and add only meaningful second-order interactions. +- Add or update matrix invariant tests before registering new dimensions. +- Keep correctness/error-injection tests separate from throughput benchmarks. +- Do not commit large generated corpora. Commit generation rules and manifests; keep only small + correctness fixtures in the repository when necessary. +- Record the exact commit, compiler, build type, host topology, command, repetitions, cache state, + and raw JSON for any claimed performance result. +- Never claim a performance improvement from a smoke run or from incomparable machines. diff --git a/be/benchmark/parquet/README.md b/be/benchmark/parquet/README.md new file mode 100644 index 00000000000000..40c79bc28bea5f --- /dev/null +++ b/be/benchmark/parquet/README.md @@ -0,0 +1,63 @@ +# Parquet reader microbenchmarks + +These benchmarks separate native page decoding from the complete local-file reader path. They use +deterministic data and verify the physical encoding recorded in each generated Parquet footer before +running a measurement. + +Agents and maintainers must read [AGENTS.md](AGENTS.md) for the exact timing boundaries, synthetic +data rules, result interpretation, current coverage limitations, and prioritized follow-up work. + +## Build + +Build the Release benchmark binary from the repository root: + +```shell +./build.sh --benchmark -j128 +``` + +List only the Parquet cases: + +```shell +be/output/lib/benchmark_test --benchmark_list_tests | grep '^Parquet' +``` + +## Decoder cases + +`ParquetDecoder` measures the native decoder with data generation and encoder setup outside the +timed region. It covers PLAIN, dictionary, byte-stream-split, and DELTA encodings across their +supported fixed-width and binary physical types. Sparse selections are provided as both one +clustered range and many alternating ranges. + +```shell +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetDecoder/plain/int64/sel_10/alternating$' \ + --benchmark_min_time=0.1s +``` + +## Local reader cases + +`ParquetReader` measures local open-to-first-block, full scan, predicate scan, and LIMIT-shaped +reads. The matrix covers: + +- PLAIN, dictionary, byte-stream-split, and DELTA binary-packed files; +- NULL ratios of 0%, 1%, 10%, 50%, and 90%, with clustered and alternating placement; +- predicate selectivities of 0%, 1%, 10%, 50%, 90%, and 100%; +- predicate-only and predicate-plus-lazy-projected reads; +- schemas with 4, 32, 128, and 512 columns, with the predicate first or last. + +Fixtures are created lazily under the system temporary directory in +`doris_parquet_reader_benchmark`. Generation, footer validation, and reader setup are excluded from +steady-state scan timings. `open_to_first_block` intentionally includes reader initialization, +footer loading, open, and the first `get_block` call. + +```shell +be/output/lib/benchmark_test \ + --benchmark_filter='^ParquetReader/predicate_scan/plain/null_50/alternating/sel_10/' \ + --benchmark_min_time=0.1s \ + --benchmark_out=parquet-reader.json \ + --benchmark_out_format=json +``` + +Every result reports throughput plus `raw_rows`, `selected_rows`, `fixture_bytes`, `ns/raw_row`, +and (when at least one row survives) `ns/selected_row`. Keep CPU frequency, build type, compiler, +machine placement, and benchmark filters fixed when comparing two commits. diff --git a/be/benchmark/parquet/benchmark_parquet_decoder.hpp b/be/benchmark/parquet/benchmark_parquet_decoder.hpp new file mode 100644 index 00000000000000..25fb05d1449e53 --- /dev/null +++ b/be/benchmark/parquet/benchmark_parquet_decoder.hpp @@ -0,0 +1,479 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/custom_allocator.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "parquet_benchmark_scenarios.h" +#include "util/faststring.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::parquet_benchmark { +namespace detail { + +constexpr size_t DECODER_ROWS = 1UL << 16; +constexpr size_t DICTIONARY_ENTRIES = 256; +constexpr size_t FIXED_BINARY_WIDTH = 16; + +struct EncodedPage { + std::vector data; + std::vector dictionary; + size_t dictionary_entries = 0; + size_t value_width = 0; + bool binary = false; +}; + +inline tparquet::Type::type physical_type(ValueType value_type) { + switch (value_type) { + case ValueType::INT32: + return tparquet::Type::INT32; + case ValueType::INT64: + return tparquet::Type::INT64; + case ValueType::FLOAT: + return tparquet::Type::FLOAT; + case ValueType::DOUBLE: + return tparquet::Type::DOUBLE; + case ValueType::BYTE_ARRAY: + return tparquet::Type::BYTE_ARRAY; + case ValueType::FIXED_LEN_BYTE_ARRAY: + return tparquet::Type::FIXED_LEN_BYTE_ARRAY; + } + throw std::logic_error("unknown Parquet benchmark value type"); +} + +inline tparquet::Encoding::type parquet_encoding(Encoding encoding) { + switch (encoding) { + case Encoding::PLAIN: + return tparquet::Encoding::PLAIN; + case Encoding::DICTIONARY: + return tparquet::Encoding::RLE_DICTIONARY; + case Encoding::BYTE_STREAM_SPLIT: + return tparquet::Encoding::BYTE_STREAM_SPLIT; + case Encoding::DELTA_BINARY_PACKED: + return tparquet::Encoding::DELTA_BINARY_PACKED; + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + return tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY; + case Encoding::DELTA_BYTE_ARRAY: + return tparquet::Encoding::DELTA_BYTE_ARRAY; + } + throw std::logic_error("unknown Parquet benchmark encoding"); +} + +inline std::shared_ptr<::parquet::ColumnDescriptor> descriptor(::parquet::Type::type type, + int type_length = -1) { + auto node = type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY + ? ::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::REQUIRED, type, + ::parquet::ConvertedType::NONE, type_length) + : ::parquet::schema::PrimitiveNode::Make( + "value", ::parquet::Repetition::REQUIRED, type); + return std::make_shared<::parquet::ColumnDescriptor>(node, 0, 0); +} + +template +std::vector fixed_values(size_t rows) { + std::vector values(rows); + for (size_t row = 0; row < rows; ++row) { + if constexpr (std::is_floating_point_v) { + values[row] = static_cast((row % 1009) * 0.25 - 100.0); + } else { + values[row] = static_cast((row * 17) % 1000003); + } + } + return values; +} + +inline std::vector binary_values(size_t rows, size_t width = FIXED_BINARY_WIDTH) { + std::vector values; + values.reserve(rows); + for (size_t row = 0; row < rows; ++row) { + std::string value(width, 'a'); + const uint64_t id = row % 1009; + memcpy(value.data(), &id, std::min(width, sizeof(id))); + values.push_back(std::move(value)); + } + return values; +} + +inline std::vector encode_plain_binary(const std::vector& values) { + size_t bytes = 0; + for (const auto& value : values) { + bytes += sizeof(uint32_t) + value.size(); + } + std::vector encoded; + encoded.reserve(bytes); + for (const auto& value : values) { + const auto length = static_cast(value.size()); + const auto* length_bytes = reinterpret_cast(&length); + encoded.insert(encoded.end(), length_bytes, length_bytes + sizeof(length)); + encoded.insert(encoded.end(), value.begin(), value.end()); + } + return encoded; +} + +template +std::vector encode_plain_fixed(const std::vector& values) { + std::vector encoded(values.size() * sizeof(T)); + memcpy(encoded.data(), values.data(), encoded.size()); + return encoded; +} + +inline std::vector encode_fixed_binary(const std::vector& values) { + std::vector encoded; + encoded.reserve(values.size() * FIXED_BINARY_WIDTH); + for (const auto& value : values) { + encoded.insert(encoded.end(), value.begin(), value.end()); + } + return encoded; +} + +inline std::vector encode_byte_stream_split(const std::vector& plain, + size_t value_width) { + const size_t rows = plain.size() / value_width; + std::vector encoded(plain.size()); + for (size_t row = 0; row < rows; ++row) { + for (size_t byte = 0; byte < value_width; ++byte) { + encoded[byte * rows + row] = plain[row * value_width + byte]; + } + } + return encoded; +} + +template +std::vector encode_delta_fixed(const std::vector& values) { + auto desc = descriptor(ParquetType::type_num); + auto encoder = ::parquet::MakeTypedEncoder( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, desc.get()); + encoder->Put(values.data(), static_cast(values.size())); + const auto buffer = encoder->FlushValues(); + return {buffer->data(), buffer->data() + buffer->size()}; +} + +inline std::vector encode_delta_binary(const std::vector& values, + ::parquet::Encoding::type encoding) { + std::vector<::parquet::ByteArray> arrays; + arrays.reserve(values.size()); + for (const auto& value : values) { + arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto desc = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = + ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, desc.get()); + encoder->Put(arrays.data(), static_cast(arrays.size())); + const auto buffer = encoder->FlushValues(); + return {buffer->data(), buffer->data() + buffer->size()}; +} + +inline std::vector encode_dictionary_indices(size_t rows) { + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 8); + for (size_t row = 0; row < rows; ++row) { + encoder.Put(static_cast(row % DICTIONARY_ENTRIES)); + } + for (size_t padding = rows; padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector result(encoded_ids.size() + 1); + result[0] = 8; + memcpy(result.data() + 1, encoded_ids.data(), encoded_ids.size()); + return result; +} + +inline EncodedPage dictionary_page(ValueType value_type) { + EncodedPage page; + page.data = encode_dictionary_indices(DECODER_ROWS); + page.dictionary_entries = DICTIONARY_ENTRIES; + page.binary = value_type == ValueType::BYTE_ARRAY; + switch (value_type) { + case ValueType::INT32: + page.value_width = sizeof(int32_t); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::INT64: + page.value_width = sizeof(int64_t); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::FLOAT: + page.value_width = sizeof(float); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::DOUBLE: + page.value_width = sizeof(double); + page.dictionary = encode_plain_fixed(fixed_values(DICTIONARY_ENTRIES)); + break; + case ValueType::BYTE_ARRAY: + page.value_width = FIXED_BINARY_WIDTH; + page.dictionary = encode_plain_binary(binary_values(DICTIONARY_ENTRIES)); + break; + case ValueType::FIXED_LEN_BYTE_ARRAY: + page.value_width = FIXED_BINARY_WIDTH; + page.dictionary = encode_fixed_binary(binary_values(DICTIONARY_ENTRIES)); + break; + } + return page; +} + +inline EncodedPage encoded_page(const DecoderScenario& scenario) { + if (scenario.encoding == Encoding::DICTIONARY) { + return dictionary_page(scenario.value_type); + } + + EncodedPage page; + page.binary = scenario.value_type == ValueType::BYTE_ARRAY; + switch (scenario.value_type) { + case ValueType::INT32: { + auto values = fixed_values(DECODER_ROWS); + page.value_width = sizeof(int32_t); + page.data = scenario.encoding == Encoding::DELTA_BINARY_PACKED + ? encode_delta_fixed<::parquet::Int32Type>(values) + : encode_plain_fixed(values); + break; + } + case ValueType::INT64: { + auto values = fixed_values(DECODER_ROWS); + page.value_width = sizeof(int64_t); + page.data = scenario.encoding == Encoding::DELTA_BINARY_PACKED + ? encode_delta_fixed<::parquet::Int64Type>(values) + : encode_plain_fixed(values); + break; + } + case ValueType::FLOAT: { + auto plain = encode_plain_fixed(fixed_values(DECODER_ROWS)); + page.value_width = sizeof(float); + page.data = scenario.encoding == Encoding::BYTE_STREAM_SPLIT + ? encode_byte_stream_split(plain, page.value_width) + : std::move(plain); + break; + } + case ValueType::DOUBLE: { + auto plain = encode_plain_fixed(fixed_values(DECODER_ROWS)); + page.value_width = sizeof(double); + page.data = scenario.encoding == Encoding::BYTE_STREAM_SPLIT + ? encode_byte_stream_split(plain, page.value_width) + : std::move(plain); + break; + } + case ValueType::BYTE_ARRAY: { + auto values = binary_values(DECODER_ROWS); + page.value_width = FIXED_BINARY_WIDTH; + if (scenario.encoding == Encoding::DELTA_LENGTH_BYTE_ARRAY) { + page.data = encode_delta_binary(values, ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY); + } else if (scenario.encoding == Encoding::DELTA_BYTE_ARRAY) { + page.data = encode_delta_binary(values, ::parquet::Encoding::DELTA_BYTE_ARRAY); + } else { + page.data = encode_plain_binary(values); + } + break; + } + case ValueType::FIXED_LEN_BYTE_ARRAY: { + auto plain = encode_fixed_binary(binary_values(DECODER_ROWS)); + page.value_width = FIXED_BINARY_WIDTH; + page.data = scenario.encoding == Encoding::BYTE_STREAM_SPLIT + ? encode_byte_stream_split(plain, page.value_width) + : std::move(plain); + break; + } + } + return page; +} + +class FixedSink final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + benchmark::DoNotOptimize(values); + benchmark::DoNotOptimize(num_values); + benchmark::DoNotOptimize(value_width); + consumed += num_values; + return Status::OK(); + } + + size_t consumed = 0; +}; + +class BinarySink final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + benchmark::DoNotOptimize(values); + benchmark::DoNotOptimize(num_values); + consumed += num_values; + return Status::OK(); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + benchmark::DoNotOptimize(encoded_data); + benchmark::DoNotOptimize(payload_offsets); + benchmark::DoNotOptimize(value_offsets); + auto value_spans_data = value_spans.data(); + benchmark::DoNotOptimize(value_spans_data); + consumed += num_values; + return Status::OK(); + } + + size_t consumed = 0; +}; + +class DictionarySink final : public ParquetDictionaryValueConsumer { +public: + DictionarySink(const uint8_t* dictionary, size_t value_width) + : _dictionary(dictionary), _value_width(value_width) {} + + Status consume_indices(const uint32_t* indices, size_t num_values) override { + for (size_t row = 0; row < num_values; ++row) { + auto* value = _dictionary + indices[row] * _value_width; + benchmark::DoNotOptimize(value); + } + consumed += num_values; + return Status::OK(); + } + + Status consume_repeated(uint32_t index, size_t num_values) override { + auto* value = _dictionary + index * _value_width; + benchmark::DoNotOptimize(value); + consumed += num_values; + return Status::OK(); + } + + size_t consumed = 0; + +private: + const uint8_t* _dictionary; + const size_t _value_width; +}; + +inline ParquetSelection native_selection(const SelectionPlan& plan) { + ParquetSelection selection { + .total_values = plan.total_rows, .selected_values = plan.selected_rows, .ranges = {}}; + selection.ranges.reserve(plan.ranges.size()); + for (const auto& range : plan.ranges) { + selection.ranges.push_back({.first = range.first, .count = range.count}); + } + return selection; +} + +inline void run_decoder(benchmark::State& state, DecoderScenario scenario, int selectivity, + Pattern pattern) { + auto page = encoded_page(scenario); + const auto plan = make_selection_plan(DECODER_ROWS, selectivity, pattern); + const auto selection = native_selection(plan); + std::unique_ptr decoder; + auto status = format::parquet::native::Decoder::get_decoder( + physical_type(scenario.value_type), parquet_encoding(scenario.encoding), decoder); + if (!status.ok()) { + state.SkipWithError(status.to_string().c_str()); + return; + } + decoder->set_type_length(static_cast(page.value_width)); + decoder->set_expected_values(DECODER_ROWS); + + std::vector dictionary_for_sink; + if (scenario.encoding == Encoding::DICTIONARY) { + dictionary_for_sink = page.dictionary; + auto dictionary = make_unique_buffer(page.dictionary.size()); + memcpy(dictionary.get(), page.dictionary.data(), page.dictionary.size()); + status = decoder->set_dict(dictionary, static_cast(page.dictionary.size()), + page.dictionary_entries); + if (!status.ok()) { + state.SkipWithError(status.to_string().c_str()); + return; + } + } + + Slice encoded(page.data.data(), page.data.size()); + FixedSink fixed_sink; + BinarySink binary_sink; + DictionarySink dictionary_sink(dictionary_for_sink.data(), page.value_width); + for (auto _ : state) { + state.PauseTiming(); + status = decoder->set_data(&encoded); + fixed_sink.consumed = 0; + binary_sink.consumed = 0; + dictionary_sink.consumed = 0; + state.ResumeTiming(); + if (scenario.encoding == Encoding::DICTIONARY) { + status = decoder->decode_selected_dictionary_values(selection, dictionary_sink); + } else if (page.binary) { + status = decoder->decode_selected_binary_values(selection, binary_sink); + } else { + status = decoder->decode_selected_fixed_values(selection, fixed_sink); + } + if (!status.ok()) { + state.SkipWithError(status.to_string().c_str()); + break; + } + benchmark::ClobberMemory(); + } + + state.SetItemsProcessed(static_cast(state.iterations()) * + static_cast(plan.selected_rows)); + state.SetBytesProcessed(static_cast(state.iterations()) * + static_cast(page.data.size())); + state.counters["raw_rows"] = static_cast(DECODER_ROWS); + state.counters["selected_rows"] = static_cast(plan.selected_rows); + state.counters["selection_ranges"] = static_cast(plan.ranges.size()); + state.counters["encoded_bytes"] = static_cast(page.data.size()); + state.counters["ns/raw_row"] = benchmark::Counter( + static_cast(DECODER_ROWS), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + if (plan.selected_rows > 0) { + state.counters["ns/selected_row"] = benchmark::Counter( + static_cast(plan.selected_rows), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + } +} + +inline bool register_decoder_benchmarks() { + for (const auto& scenario : decoder_scenarios()) { + for (const int selectivity : {1, 10, 50, 100}) { + for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { + const std::string name = "ParquetDecoder/" + to_string(scenario.encoding) + "/" + + to_string(scenario.value_type) + "/sel_" + + std::to_string(selectivity) + "/" + to_string(pattern); + benchmark::RegisterBenchmark(name.c_str(), [=](benchmark::State& state) { + run_decoder(state, scenario, selectivity, pattern); + })->Unit(benchmark::kNanosecond); + } + } + } + return true; +} + +inline const bool DECODER_BENCHMARKS_REGISTERED = register_decoder_benchmarks(); + +} // namespace detail +} // namespace doris::parquet_benchmark diff --git a/be/benchmark/parquet/benchmark_parquet_reader.hpp b/be/benchmark/parquet/benchmark_parquet_reader.hpp new file mode 100644 index 00000000000000..ae3e14c0f66ad9 --- /dev/null +++ b/be/benchmark/parquet/benchmark_parquet_reader.hpp @@ -0,0 +1,448 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_reader.h" +#include "gen_cpp/Types_types.h" +#include "io/io_common.h" +#include "parquet_benchmark_scenarios.h" +#include "runtime/runtime_state.h" +#include "storage/index/zone_map/zonemap_eval_context.h" +#include "storage/index/zone_map/zonemap_filter_result.h" + +namespace doris::parquet_benchmark { +namespace reader_detail { + +constexpr size_t READER_ROWS = 1UL << 14; +constexpr size_t READER_ROW_GROUP_ROWS = 1UL << 12; + +inline void throw_if_error(const Status& status) { + if (!status.ok()) { + throw std::runtime_error(status.to_string()); + } +} + +inline bool is_null_row(size_t row, int null_percent, Pattern pattern) { + if (null_percent <= 0) { + return false; + } + if (pattern == Pattern::ALTERNATING) { + constexpr size_t NULL_PERIOD = 101; + const size_t null_slots = (static_cast(null_percent) * NULL_PERIOD + 99) / 100; + return (row * 37) % NULL_PERIOD < null_slots; + } + constexpr size_t CLUSTER_ROWS = 1024; + return row % CLUSTER_ROWS < CLUSTER_ROWS * static_cast(null_percent) / 100; +} + +inline std::shared_ptr build_int32_array(int null_percent, Pattern pattern) { + arrow::Int32Builder builder; + PARQUET_THROW_NOT_OK(builder.Reserve(READER_ROWS)); + for (size_t row = 0; row < READER_ROWS; ++row) { + if (is_null_row(row, null_percent, pattern)) { + PARQUET_THROW_NOT_OK(builder.AppendNull()); + } else { + PARQUET_THROW_NOT_OK(builder.Append(static_cast(row % 100))); + } + } + return builder.Finish().ValueOrDie(); +} + +inline ::parquet::Encoding::type file_encoding(Encoding encoding) { + switch (encoding) { + case Encoding::PLAIN: + return ::parquet::Encoding::PLAIN; + case Encoding::BYTE_STREAM_SPLIT: + return ::parquet::Encoding::BYTE_STREAM_SPLIT; + case Encoding::DELTA_BINARY_PACKED: + return ::parquet::Encoding::DELTA_BINARY_PACKED; + case Encoding::DICTIONARY: + return ::parquet::Encoding::RLE_DICTIONARY; + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + return ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY; + case Encoding::DELTA_BYTE_ARRAY: + return ::parquet::Encoding::DELTA_BYTE_ARRAY; + } + throw std::logic_error("unknown Parquet benchmark encoding"); +} + +inline std::string fixture_name(const ReaderScenario& scenario) { + return "v2_" + to_string(scenario.encoding) + "_null" + std::to_string(scenario.null_percent) + + "_" + to_string(scenario.null_pattern) + "_w" + std::to_string(scenario.schema_width) + + "_p" + std::to_string(scenario.predicate_position) + ".parquet"; +} + +inline void verify_fixture_encoding(const std::filesystem::path& path, + const ReaderScenario& scenario) { + auto reader = ::parquet::ParquetFileReader::OpenFile(path.string(), false); + const auto metadata = reader->metadata(); + if (metadata->num_rows() != static_cast(READER_ROWS) || + metadata->num_columns() != scenario.schema_width) { + throw std::runtime_error("Parquet benchmark fixture has unexpected shape: " + + path.string()); + } + const auto expected = file_encoding(scenario.encoding); + for (int row_group = 0; row_group < metadata->num_row_groups(); ++row_group) { + for (int column = 0; column < metadata->num_columns(); ++column) { + const auto encodings = metadata->RowGroup(row_group)->ColumnChunk(column)->encodings(); + if (std::ranges::find(encodings, expected) == encodings.end()) { + throw std::runtime_error("Parquet benchmark fixture did not use " + + to_string(scenario.encoding) + + " encoding: " + path.string()); + } + } + } +} + +inline std::filesystem::path ensure_fixture(const ReaderScenario& scenario) { + static std::mutex fixture_mutex; + const auto directory = + std::filesystem::temp_directory_path() / "doris_parquet_reader_benchmark"; + const auto path = directory / fixture_name(scenario); + std::lock_guard guard(fixture_mutex); + if (std::filesystem::exists(path)) { + verify_fixture_encoding(path, scenario); + return path; + } + + std::filesystem::create_directories(directory); + const auto temporary_path = path.string() + ".tmp"; + std::filesystem::remove(temporary_path); + const auto values = build_int32_array(scenario.null_percent, scenario.null_pattern); + std::vector> fields; + std::vector> columns; + fields.reserve(scenario.schema_width); + columns.reserve(scenario.schema_width); + for (int column = 0; column < scenario.schema_width; ++column) { + fields.push_back(arrow::field("c" + std::to_string(column), arrow::int32(), true)); + columns.push_back(std::make_shared(values)); + } + const auto table = arrow::Table::Make(arrow::schema(std::move(fields)), std::move(columns)); + + const auto output_result = arrow::io::FileOutputStream::Open(temporary_path); + if (!output_result.ok()) { + throw std::runtime_error(output_result.status().ToString()); + } + const auto output = *output_result; + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + properties.disable_statistics(); + if (scenario.encoding == Encoding::DICTIONARY) { + properties.enable_dictionary(); + } else { + properties.disable_dictionary(); + properties.encoding(file_encoding(scenario.encoding)); + } + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, + READER_ROW_GROUP_ROWS, properties.build())); + PARQUET_THROW_NOT_OK(output->Close()); + std::filesystem::rename(temporary_path, path); + verify_fixture_encoding(path, scenario); + return path; +} + +class Int32LessThanExpr final : public VExpr { +public: + Int32LessThanExpr(int column_id, int32_t upper_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _upper_bound(upper_bound) {} + + Status execute_column_impl(VExprContext*, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& nullable = + assert_cast(*block->get_by_position(_column_id).column); + const auto& values = assert_cast(nullable.get_nested_column()); + const auto& nulls = nullable.get_null_map_data(); + auto result = ColumnUInt8::create(count, 0); + auto& matches = result->get_data(); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + matches[row] = !nulls[input_row] && values.get_element(input_row) < _upper_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_INT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int column_id, + uint8_t* matches) const override { + DORIS_CHECK(column_id == _column_id); + DORIS_CHECK(value_width == sizeof(int32_t)); + const auto* typed_values = reinterpret_cast(values); + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= typed_values[row] < _upper_bound; + } + return Status::OK(); + } + + bool can_evaluate_zonemap_filter() const override { return true; } + + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(_column_id); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + const auto upper_bound = Field::create_field(_upper_bound); + return zone_map->min_value >= upper_bound ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + const auto* dictionary = ctx.slot(_column_id); + if (dictionary == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + const auto upper_bound = Field::create_field(_upper_bound); + return std::ranges::any_of(dictionary->values, + [&](const Field& value) { return value < upper_bound; }) + ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const int32_t _upper_bound; + const std::string _expr_name = "ParquetBenchmarkInt32LessThan"; +}; + +inline VExprContextSPtr make_predicate(int column_position, int selectivity_percent) { + auto context = VExprContext::create_shared( + std::make_shared(column_position, selectivity_percent)); + context->_prepared = true; + context->_opened = true; + return context; +} + +inline Block make_block(const std::vector& schema) { + Block block; + for (const auto& column : schema) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +struct ReaderSession { + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + std::unique_ptr reader; + std::vector schema; + std::shared_ptr request; +}; + +inline std::unique_ptr open_reader(const std::filesystem::path& path, + const ReaderScenario& scenario) { + auto session = std::make_unique(); + auto properties = std::make_shared(); + properties->system_type = TFileType::FILE_LOCAL; + auto description = std::make_unique(); + description->path = path.string(); + description->file_size = static_cast(std::filesystem::file_size(path)); + description->range_start_offset = 0; + description->range_size = -1; + session->reader = std::make_unique(properties, description, + nullptr, nullptr); + throw_if_error(session->reader->init(&session->runtime_state)); + throw_if_error(session->reader->get_schema(&session->schema)); + + session->request = std::make_shared(); + format::FileScanRequestBuilder request_builder(session->request.get()); + if (scenario.operation == ReaderOperation::FULL_SCAN) { + for (int column = 0; column < scenario.schema_width; ++column) { + throw_if_error(request_builder.add_non_predicate_column(format::LocalColumnId(column))); + } + } else if (scenario.operation == ReaderOperation::PREDICATE_SCAN) { + const auto predicate_id = format::LocalColumnId(scenario.predicate_position); + throw_if_error(request_builder.add_predicate_column(predicate_id)); + if (scenario.projection == Projection::PREDICATE_ONLY) { + session->request->predicate_only_columns.push_back(predicate_id); + } else { + const int payload = scenario.predicate_position == 0 ? scenario.schema_width - 1 : 0; + throw_if_error( + request_builder.add_non_predicate_column(format::LocalColumnId(payload))); + } + const auto predicate_position = session->request->local_positions.at(predicate_id).value(); + session->request->conjuncts.push_back( + make_predicate(static_cast(predicate_position), scenario.selectivity_percent)); + } else { + throw_if_error(request_builder.add_non_predicate_column(format::LocalColumnId(0))); + if (scenario.schema_width > 1) { + throw_if_error(request_builder.add_non_predicate_column(format::LocalColumnId(1))); + } + } + + if (scenario.operation == ReaderOperation::LIMIT_1) { + session->reader->set_batch_size(1); + } else if (scenario.operation == ReaderOperation::LIMIT_1000) { + session->reader->set_batch_size(1000); + } + throw_if_error(session->reader->open(session->request)); + return session; +} + +inline size_t scan_reader(ReaderSession* session, const ReaderScenario& scenario) { + size_t output_rows = 0; + bool eof = false; + const size_t limit = scenario.operation == ReaderOperation::LIMIT_1 ? 1 + : scenario.operation == ReaderOperation::LIMIT_1000 ? 1000 + : 0; + while (!eof) { + auto block = make_block(session->schema); + size_t rows = 0; + throw_if_error(session->reader->get_block(&block, &rows, &eof)); + output_rows += rows; + benchmark::DoNotOptimize(block); + if (scenario.operation == ReaderOperation::OPEN_TO_FIRST_BLOCK) { + break; + } + if (limit != 0 && output_rows >= limit) { + break; + } + } + return output_rows; +} + +inline size_t raw_rows_per_iteration(const ReaderScenario& scenario) { + if (scenario.operation == ReaderOperation::LIMIT_1) { + return 1; + } + if (scenario.operation == ReaderOperation::LIMIT_1000) { + return 1000; + } + if (scenario.operation == ReaderOperation::OPEN_TO_FIRST_BLOCK) { + return format::parquet::ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; + } + return READER_ROWS; +} + +inline int projected_columns(const ReaderScenario& scenario) { + if (scenario.operation == ReaderOperation::FULL_SCAN) { + return scenario.schema_width; + } + if (scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.projection == Projection::PREDICATE_ONLY) { + return 1; + } + return std::min(2, scenario.schema_width); +} + +inline void run_reader(benchmark::State& state, ReaderScenario scenario) { + std::filesystem::path fixture; + try { + fixture = ensure_fixture(scenario); + size_t selected_rows = 0; + for (auto _ : state) { + if (scenario.operation != ReaderOperation::OPEN_TO_FIRST_BLOCK) { + state.PauseTiming(); + } + auto session = open_reader(fixture, scenario); + if (scenario.operation != ReaderOperation::OPEN_TO_FIRST_BLOCK) { + state.ResumeTiming(); + } + selected_rows = scan_reader(session.get(), scenario); + state.PauseTiming(); + throw_if_error(session->reader->close()); + state.ResumeTiming(); + benchmark::ClobberMemory(); + } + + const auto raw_rows = raw_rows_per_iteration(scenario); + state.SetItemsProcessed(static_cast(state.iterations() * selected_rows)); + state.SetBytesProcessed(static_cast( + state.iterations() * raw_rows * projected_columns(scenario) * sizeof(int32_t))); + state.counters["raw_rows"] = static_cast(raw_rows); + state.counters["selected_rows"] = static_cast(selected_rows); + state.counters["fixture_bytes"] = static_cast(std::filesystem::file_size(fixture)); + state.counters["ns/raw_row"] = benchmark::Counter( + static_cast(raw_rows), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + if (selected_rows > 0) { + state.counters["ns/selected_row"] = benchmark::Counter( + static_cast(selected_rows), + benchmark::Counter::kIsIterationInvariantRate | benchmark::Counter::kInvert); + } + } catch (const std::exception& error) { + state.SkipWithError(error.what()); + } +} + +inline bool register_reader_benchmarks() { + for (const auto& scenario : reader_scenarios()) { + std::string name = + "ParquetReader/" + to_string(scenario.operation) + "/" + + to_string(scenario.encoding) + "/null_" + std::to_string(scenario.null_percent) + + "/" + to_string(scenario.null_pattern) + "/sel_" + + std::to_string(scenario.selectivity_percent) + "/" + + to_string(scenario.projection) + "/width_" + std::to_string(scenario.schema_width) + + "/predicate_" + std::to_string(scenario.predicate_position); + benchmark::RegisterBenchmark(name.c_str(), [=](benchmark::State& state) { + run_reader(state, scenario); + })->Unit(benchmark::kNanosecond); + } + return true; +} + +inline const bool READER_BENCHMARKS_REGISTERED = register_reader_benchmarks(); + +} // namespace reader_detail +} // namespace doris::parquet_benchmark diff --git a/be/benchmark/parquet/parquet_benchmark_scenarios.h b/be/benchmark/parquet/parquet_benchmark_scenarios.h new file mode 100644 index 00000000000000..a01c955c6a2aed --- /dev/null +++ b/be/benchmark/parquet/parquet_benchmark_scenarios.h @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace doris::parquet_benchmark { + +enum class Encoding { + PLAIN, + DICTIONARY, + BYTE_STREAM_SPLIT, + DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY +}; +enum class ValueType { INT32, INT64, FLOAT, DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY }; +enum class Pattern { CLUSTERED, ALTERNATING }; +enum class Projection { PREDICATE_ONLY, PREDICATE_PROJECTED }; +enum class ReaderOperation { OPEN_TO_FIRST_BLOCK, FULL_SCAN, PREDICATE_SCAN, LIMIT_1, LIMIT_1000 }; + +struct DecoderScenario { + Encoding encoding; + ValueType value_type; +}; + +struct ReaderScenario { + ReaderOperation operation; + Encoding encoding; + int null_percent; + Pattern null_pattern; + int selectivity_percent; + Projection projection; + int schema_width; + int predicate_position; +}; + +struct SelectionRange { + size_t first; + size_t count; +}; + +struct SelectionPlan { + size_t total_rows = 0; + size_t selected_rows = 0; + std::vector ranges; +}; + +inline std::vector decoder_scenarios() { + return { + {Encoding::PLAIN, ValueType::INT32}, + {Encoding::PLAIN, ValueType::INT64}, + {Encoding::PLAIN, ValueType::FLOAT}, + {Encoding::PLAIN, ValueType::DOUBLE}, + {Encoding::PLAIN, ValueType::BYTE_ARRAY}, + {Encoding::PLAIN, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::INT32}, + {Encoding::DICTIONARY, ValueType::INT64}, + {Encoding::DICTIONARY, ValueType::FLOAT}, + {Encoding::DICTIONARY, ValueType::DOUBLE}, + {Encoding::DICTIONARY, ValueType::BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FLOAT}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::DOUBLE}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT32}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT64}, + {Encoding::DELTA_LENGTH_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + {Encoding::DELTA_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + }; +} + +inline std::vector reader_scenarios() { + std::vector scenarios; + std::set> seen; + const auto add = [&](ReaderScenario scenario) { + const auto key = std::make_tuple(scenario.operation, scenario.encoding, + scenario.null_percent, scenario.null_pattern, + scenario.selectivity_percent, scenario.projection, + scenario.schema_width, scenario.predicate_position); + if (seen.insert(key).second) { + scenarios.push_back(scenario); + } + }; + + const ReaderScenario baseline {.operation = ReaderOperation::FULL_SCAN, + .encoding = Encoding::PLAIN, + .null_percent = 10, + .null_pattern = Pattern::ALTERNATING, + .selectivity_percent = 10, + .projection = Projection::PREDICATE_PROJECTED, + .schema_width = 32, + .predicate_position = 0}; + for (const auto operation : + {ReaderOperation::OPEN_TO_FIRST_BLOCK, ReaderOperation::FULL_SCAN, + ReaderOperation::PREDICATE_SCAN, ReaderOperation::LIMIT_1, ReaderOperation::LIMIT_1000}) { + auto scenario = baseline; + scenario.operation = operation; + add(scenario); + } + for (const auto encoding : {Encoding::PLAIN, Encoding::DICTIONARY, Encoding::BYTE_STREAM_SPLIT, + Encoding::DELTA_BINARY_PACKED}) { + auto scenario = baseline; + scenario.encoding = encoding; + add(scenario); + scenario.operation = ReaderOperation::PREDICATE_SCAN; + add(scenario); + } + for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) { + for (const int selectivity : {1, 10, 50, 90}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + auto scenario = baseline; + scenario.operation = ReaderOperation::PREDICATE_SCAN; + scenario.encoding = encoding; + scenario.selectivity_percent = selectivity; + scenario.projection = projection; + add(scenario); + } + } + } + for (const int width : {4, 32, 128, 512}) { + for (const int predicate_position : {0, width - 1}) { + auto scenario = baseline; + scenario.operation = ReaderOperation::PREDICATE_SCAN; + scenario.schema_width = width; + scenario.predicate_position = predicate_position; + add(scenario); + } + } + for (const int null_percent : {0, 1, 10, 50, 90}) { + for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { + for (const int selectivity : {0, 1, 10, 50, 90, 100}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + auto scenario = baseline; + scenario.operation = ReaderOperation::PREDICATE_SCAN; + scenario.null_percent = null_percent; + scenario.null_pattern = pattern; + scenario.selectivity_percent = selectivity; + scenario.projection = projection; + add(scenario); + } + } + } + } + return scenarios; +} + +inline SelectionPlan make_selection_plan(size_t total_rows, int selectivity_percent, + Pattern pattern) { + SelectionPlan plan {.total_rows = total_rows, .selected_rows = 0, .ranges = {}}; + if (total_rows == 0 || selectivity_percent <= 0) { + return plan; + } + if (selectivity_percent >= 100) { + plan.selected_rows = total_rows; + plan.ranges.push_back({.first = 0, .count = total_rows}); + return plan; + } + plan.selected_rows = total_rows * static_cast(selectivity_percent) / 100; + if (plan.selected_rows == 0) { + plan.selected_rows = 1; + } + if (pattern == Pattern::CLUSTERED) { + plan.ranges.push_back({.first = 0, .count = plan.selected_rows}); + return plan; + } + + // Evenly spaced rows deliberately maximize the number of physical ranges. This is the + // adversarial sparse shape that exposes per-run decoder and cursor overhead. + for (size_t selected = 0; selected < plan.selected_rows; ++selected) { + const size_t row = selected * total_rows / plan.selected_rows; + if (!plan.ranges.empty() && plan.ranges.back().first + plan.ranges.back().count == row) { + ++plan.ranges.back().count; + } else { + plan.ranges.push_back({.first = row, .count = 1}); + } + } + return plan; +} + +inline std::string to_string(Encoding value) { + switch (value) { + case Encoding::PLAIN: + return "plain"; + case Encoding::DICTIONARY: + return "dictionary"; + case Encoding::BYTE_STREAM_SPLIT: + return "byte_stream_split"; + case Encoding::DELTA_BINARY_PACKED: + return "delta_binary_packed"; + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + return "delta_length_byte_array"; + case Encoding::DELTA_BYTE_ARRAY: + return "delta_byte_array"; + } + return "unknown"; +} + +inline std::string to_string(ValueType value) { + switch (value) { + case ValueType::INT32: + return "int32"; + case ValueType::INT64: + return "int64"; + case ValueType::FLOAT: + return "float"; + case ValueType::DOUBLE: + return "double"; + case ValueType::BYTE_ARRAY: + return "byte_array"; + case ValueType::FIXED_LEN_BYTE_ARRAY: + return "fixed_len_byte_array"; + } + return "unknown"; +} + +inline std::string to_string(Pattern value) { + return value == Pattern::CLUSTERED ? "clustered" : "alternating"; +} + +inline std::string to_string(Projection value) { + return value == Projection::PREDICATE_ONLY ? "predicate_only" : "predicate_projected"; +} + +inline std::string to_string(ReaderOperation value) { + switch (value) { + case ReaderOperation::OPEN_TO_FIRST_BLOCK: + return "open_to_first_block"; + case ReaderOperation::FULL_SCAN: + return "full_scan"; + case ReaderOperation::PREDICATE_SCAN: + return "predicate_scan"; + case ReaderOperation::LIMIT_1: + return "limit_1"; + case ReaderOperation::LIMIT_1000: + return "limit_1000"; + } + return "unknown"; +} + +} // namespace doris::parquet_benchmark diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index f8598d785dfed5..92dffd98da875b 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -110,6 +110,10 @@ add_thirdparty(arrow_flight_sql LIB64) add_thirdparty(arrow_dataset LIB64) add_thirdparty(arrow_acero LIB64) add_thirdparty(parquet LIB64) +# liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it +# so the final linker resolves C math symbols from the system library first. +add_thirdparty(lance_c LIB64 NOTADD) +list(APPEND COMMON_THIRDPARTY m lance_c) add_thirdparty(brpc LIB64) add_thirdparty(rocksdb) add_thirdparty(cyrus-sasl LIBNAME "lib/libsasl2.a") diff --git a/be/src/common/compiler_util.h b/be/src/common/compiler_util.h index 559fca903e630e..1d2a25b7742c91 100644 --- a/be/src/common/compiler_util.h +++ b/be/src/common/compiler_util.h @@ -56,4 +56,12 @@ #define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined"))) #else #define NO_SANITIZE_UNDEFINED -#endif \ No newline at end of file +#endif + +// Allow reassociation only when the evaluation order is not part of the operation's contract. +// This may change the least significant bits of floating-point results. +#ifdef __clang__ +#define ALLOW_FP_REASSOCIATION _Pragma("clang fp reassociate(on)") +#else +#define ALLOW_FP_REASSOCIATION +#endif diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index ca20cb63990447..3955878e2761f0 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -467,6 +467,10 @@ DEFINE_mDouble(sparse_column_compaction_threshold_percent, "0.05"); // Enable RLE batch Put optimization for compaction DEFINE_mBool(enable_rle_batch_put_optimization, "true"); +// Enable PDEP-based bit unpacking. Disable it on CPUs where PDEP is microcoded and slower than +// the scalar implementation, such as AMD Zen+ and Zen 2. +DEFINE_Bool(enable_bmi2_optimizations, "true"); + // If enabled, segments will be flushed column by column DEFINE_mBool(enable_vertical_segment_writer, "true"); @@ -722,7 +726,7 @@ DEFINE_Int32(fragment_mgr_async_work_pool_queue_size, "4096"); // The read size is the size of the reads sent to os. // There is a trade off of latency and throughout, trying to keep disks busy but -// not introduce seeks. The literature seems to agree that with 8 MB reads, random +// not introduce seeks. The literature seems to agree that with 8 MB reads, random // io and sequential io perform similarly. DEFINE_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in bytes) @@ -1446,6 +1450,9 @@ DEFINE_mInt32(group_commit_queue_mem_limit, "67108864"); // group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified. DEFINE_String(group_commit_wal_max_disk_limit, "10%"); DEFINE_Bool(group_commit_wait_replay_wal_finish, "false"); +// Max WAL count for one table before rejecting async group commit loads. +// 0 means no limit. +DEFINE_mInt32(group_commit_max_wal_num_per_table, "10"); // Max time(ms) to wait for creating group commit plan fragment. // 0 means no timeout, default 2min. DEFINE_mInt32(group_commit_create_plan_timeout_ms, "120000"); @@ -1539,6 +1546,9 @@ DEFINE_mInt64(s3_put_token_per_second, "1000000000000000000"); DEFINE_Validator(s3_put_token_per_second, [](int64_t config) -> bool { return config > 0; }); DEFINE_mInt64(s3_put_token_limit, "0"); +// Log active S3 rate limiter every N throttled/rejected requests, 0 means no log. +DEFINE_mInt64(s3_rate_limiter_log_interval, "1000"); +DEFINE_Validator(s3_rate_limiter_log_interval, [](int64_t config) -> bool { return config >= 0; }); DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/connectors"); @@ -1701,6 +1711,12 @@ DEFINE_mBool(enable_pipeline_task_leakage_detect, "false"); DEFINE_mInt32(check_score_rounds_num, "1000"); DEFINE_Int32(query_cache_size, "512"); +// Max number of incremental merges accumulated on one query cache entry before +// a full recompute is forced. Each incremental merge appends the delta partial +// blocks to the entry, so the entry gets more fragmented (and the upstream merge +// aggregation does more work) as deltas accumulate; a periodic full recompute +// compacts the entry back to a minimal set of blocks. +DEFINE_mInt32(query_cache_max_incremental_merge_count, "8"); // Enable validation to check the correctness of table size. DEFINE_Bool(enable_table_size_correctness_check, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 2a6b21ccf8c0e3..cb09dfcb538cff 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -536,6 +536,7 @@ DECLARE_mInt64(vertical_compaction_max_segment_size); DECLARE_mDouble(sparse_column_compaction_threshold_percent); // Enable RLE batch Put optimization for compaction DECLARE_mBool(enable_rle_batch_put_optimization); +DECLARE_Bool(enable_bmi2_optimizations); // If enabled, segments will be flushed column by column DECLARE_mBool(enable_vertical_segment_writer); @@ -1527,6 +1528,8 @@ DECLARE_mInt32(group_commit_queue_mem_limit); // group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified. DECLARE_mString(group_commit_wal_max_disk_limit); DECLARE_Bool(group_commit_wait_replay_wal_finish); +// Max WAL count for one table before rejecting async group commit loads. 0 means no limit. +DECLARE_mInt32(group_commit_max_wal_num_per_table); // Max time(ms) to wait for creating group commit plan fragment. 0 means no timeout. DECLARE_mInt32(group_commit_create_plan_timeout_ms); @@ -1617,6 +1620,7 @@ DECLARE_mInt64(s3_get_token_limit); DECLARE_mInt64(s3_put_bucket_tokens); DECLARE_mInt64(s3_put_token_per_second); DECLARE_mInt64(s3_put_token_limit); +DECLARE_mInt64(s3_rate_limiter_log_interval); // max s3 client retry times DECLARE_mInt32(max_s3_client_retry); // When meet s3 429 error, the "get" request will @@ -1758,6 +1762,9 @@ DECLARE_mInt32(check_score_rounds_num); // MB DECLARE_Int32(query_cache_size); +// Max incremental merges on one query cache entry before forcing a full +// recompute to compact the entry (see query_cache.h QueryCacheRuntime). +DECLARE_mInt32(query_cache_max_incremental_merge_count); DECLARE_Bool(force_regenerate_rowsetid_on_start_error); // Enable validation to check the correctness of table size. diff --git a/be/src/common/metrics/doris_metrics.cpp b/be/src/common/metrics/doris_metrics.cpp index 85317133e12382..de596407e512b3 100644 --- a/be/src/common/metrics/doris_metrics.cpp +++ b/be/src/common/metrics/doris_metrics.cpp @@ -48,6 +48,15 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(query_scan_bytes_from_local, MetricUnit::BY DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(query_scan_bytes_from_remote, MetricUnit::BYTES); DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(query_scan_rows, MetricUnit::ROWS); DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(query_scan_count, MetricUnit::NOUNIT); +DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_stale_hit_total, MetricUnit::REQUESTS, + "Query cache decisions that reused a stale entry by " + "incremental merge."); +DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_incremental_fallback_total, MetricUnit::REQUESTS, + "Query cache decisions that could have merged incrementally " + "but fell back to a full recompute."); +DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_write_back_total, MetricUnit::REQUESTS, + "Query cache entries handed to the cache to be written back " + "(full or merged); admission may still turn one down."); DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(push_requests_success_total, MetricUnit::REQUESTS, "", push_requests_total, Labels({{"status", "SUCCESS"}})); DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(push_requests_fail_total, MetricUnit::REQUESTS, "", @@ -291,6 +300,9 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_scan_bytes_from_local); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_scan_bytes_from_remote); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_scan_rows); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_stale_hit_total); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_incremental_fallback_total); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_write_back_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, push_requests_success_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, push_requests_fail_total); diff --git a/be/src/common/metrics/doris_metrics.h b/be/src/common/metrics/doris_metrics.h index 764d5dd956a490..1679f28d775a8d 100644 --- a/be/src/common/metrics/doris_metrics.h +++ b/be/src/common/metrics/doris_metrics.h @@ -54,6 +54,17 @@ class DorisMetrics { IntCounter* query_scan_bytes_from_remote = nullptr; IntCounter* query_scan_rows = nullptr; + // Query cache incremental merge (see runtime/query_cache/query_cache.h): + // how many instance decisions reused a stale entry incrementally, how many + // could have but fell back to a full recompute, and how many entries were + // handed to the cache to be written back (the cache may still turn one + // down: its LRU-K admission only keeps a key that comes back while the + // shard is full). Per-query breakdown lives in the profile + // (HitCacheStale / IncrementalFallbackReason / InsertCache). + IntCounter* query_cache_stale_hit_total = nullptr; + IntCounter* query_cache_incremental_fallback_total = nullptr; + IntCounter* query_cache_write_back_total = nullptr; + IntCounter* push_requests_success_total = nullptr; IntCounter* push_requests_fail_total = nullptr; IntCounter* push_request_duration_us = nullptr; diff --git a/be/src/common/phdr_cache.h b/be/src/common/phdr_cache.h index da18b698f0d18b..abf08a0500f8c0 100644 --- a/be/src/common/phdr_cache.h +++ b/be/src/common/phdr_cache.h @@ -75,5 +75,7 @@ class ScopedPHDRCacheRead { ScopedPHDRCacheRead& operator=(const ScopedPHDRCacheRead&) = delete; private: +#if defined(__linux__) && !defined(THREAD_SANITIZER) && !defined(USE_MUSL) bool _previous = false; +#endif }; diff --git a/be/src/common/stack_trace.cpp b/be/src/common/stack_trace.cpp index 9ff2b3e79318a5..8ab6d76414a6e1 100644 --- a/be/src/common/stack_trace.cpp +++ b/be/src/common/stack_trace.cpp @@ -44,6 +44,7 @@ #include "common/memory_sanitizer.h" #include "common/symbol_index.h" #include "exec/common/hex.h" +#include "util/debug_points.h" #include "util/string_util.h" namespace { @@ -304,11 +305,17 @@ StackTrace::StackTrace(const ucontext_t& signal_context) { } void StackTrace::tryCapture() { + int frame_count = 0; #if defined(USE_UNWIND) && USE_UNWIND && defined(__x86_64__) - size = unw_backtrace(frame_pointers.data(), capacity); + frame_count = unw_backtrace(frame_pointers.data(), capacity); #else - size = backtrace(frame_pointers.data(), capacity); + frame_count = backtrace(frame_pointers.data(), capacity); #endif + size = frame_count > 0 ? static_cast(frame_count) : 0; + if (doris::config::enable_debug_points && + doris::DebugPoints::instance()->is_enable("StackTrace::tryCapture.empty_trace")) { + size = 0; + } __msan_unpoison(frame_pointers.data(), size * sizeof(frame_pointers[0])); } @@ -480,11 +487,17 @@ std::string StackTrace::toString(int start_pointers_index, const std::string& dwarf_location_info_mode) const { // Default delete the first three frame pointers, which are inside the stack_trace.cpp. start_pointers_index += 3; + if (start_pointers_index < 0 || static_cast(start_pointers_index) >= size) { + // Unwind can legitimately return fewer frames than the internal frames we normally hide. + return toStringCached({}, 0, 0, dwarf_location_info_mode); + } + + const auto first_frame = static_cast(start_pointers_index); StackTrace::FramePointers frame_pointers_raw {}; - std::copy(frame_pointers.begin() + start_pointers_index, frame_pointers.end(), + std::copy(frame_pointers.begin() + first_frame, frame_pointers.end(), frame_pointers_raw.begin()); - return toStringCached(frame_pointers_raw, offset, size - start_pointers_index, - dwarf_location_info_mode); + return toStringCached(frame_pointers_raw, offset > first_frame ? offset - first_frame : 0, + size - first_frame, dwarf_location_info_mode); } std::string StackTrace::toString(void** frame_pointers_raw, size_t offset, size_t size, diff --git a/be/src/core/block/block.h b/be/src/core/block/block.h index 6031be051f9ec1..b45d4bd7d18a45 100644 --- a/be/src/core/block/block.h +++ b/be/src/core/block/block.h @@ -448,6 +448,15 @@ class MutableBlock { DataTypes _data_types; std::vector _names; + void materialize_const_column(size_t position) { + if (is_column_const(*_columns[position])) { + // ScopedMutableBlock can retain a const destination while merge materializes + // its source, so normalize the destination before appending full columns. + _columns[position] = + IColumn::mutate(_columns[position]->convert_to_full_column_if_const()); + } + } + public: // Build from a consumed Block. This has no restore contract: the source // Block is left without columns and must not be used as a live output block. @@ -587,6 +596,7 @@ class MutableBlock { dump_names(), dump_types(), block.dump_names(), block.dump_types()); } + materialize_const_column(i); _columns[i]->insert_range_from_ignore_overflow( *block.get_by_position(i).column->convert_to_full_column_if_const().get(), 0, block.rows()); @@ -620,6 +630,7 @@ class MutableBlock { block.dump_names(), block.dump_types()); } for (int i = 0; i < _columns.size(); ++i) { + materialize_const_column(i); if (!_data_types[i]->equals(*block.get_by_position(i).type)) { DCHECK(_data_types[i]->is_nullable()) << " target type: " << _data_types[i]->get_name() diff --git a/be/src/core/column/column_string.h b/be/src/core/column/column_string.h index 9ca55caa9c6aa5..616d6ef9df39e1 100644 --- a/be/src/core/column/column_string.h +++ b/be/src/core/column/column_string.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -276,6 +277,56 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + template + void insert_many_parquet_plain_byte_arrays(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num, + const std::vector& value_spans) { + if (UNLIKELY(num == 0)) { + return; + } + const size_t old_chars_size = chars.size(); + const size_t bytes = value_offsets[num]; + check_chars_length(old_chars_size + bytes, offsets.size() + num); + chars.resize(old_chars_size + bytes); + + // Source payloads may be contiguous for decoder-owned buffers even though PLAIN normally + // separates them with length prefixes. Coalesce whenever the published layout permits it; + // the fallback remains one direct source-to-column copy without a StringRef staging array. + size_t covered_values = 0; + for (const auto& span : value_spans) { + DORIS_CHECK_EQ(span.first, covered_values); + DORIS_CHECK_LE(span.first + span.count, num); + size_t run_first = span.first; + const size_t span_end = span.first + span.count; + while (run_first < span_end) { + size_t run_end = run_first + 1; + while (run_end < span_end && + payload_offsets[run_end] == payload_offsets[run_first] + + value_offsets[run_end] - + value_offsets[run_first]) { + ++run_end; + } + const size_t run_bytes = value_offsets[run_end] - value_offsets[run_first]; + if (run_bytes != 0) { + memcpy(chars.data() + old_chars_size + value_offsets[run_first], + encoded_data + payload_offsets[run_first], run_bytes); + } + run_first = run_end; + } + covered_values = span_end; + } + DORIS_CHECK_EQ(covered_values, num); + + const size_t old_rows = offsets.size(); + const auto tail_offset = offsets.back(); + offsets.resize(old_rows + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_rows + row] = tail_offset + value_offsets[row + 1]; + } + sanity_check_simple(); + } + // Insert `num` string entries with real length information but no actual // character data. The `lengths` array provides the byte length of each // string. Offsets are built with correct cumulative sizes so that @@ -322,6 +373,35 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + void insert_many_fixed_length_data(const char* data, size_t value_length, size_t num) { + if (num == 0) { + return; + } + if (value_length > std::numeric_limits::max() / num) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + const size_t old_chars_size = chars.size(); + const size_t old_offsets_size = offsets.size(); + const size_t bytes = value_length * num; + if (bytes > std::numeric_limits::max() - old_chars_size || + num > std::numeric_limits::max() - old_offsets_size) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + check_chars_length(old_chars_size + bytes, old_offsets_size + num); + chars.resize(old_chars_size + bytes); + if (bytes != 0) { + memcpy(chars.data() + old_chars_size, data, bytes); + } + offsets.resize(old_offsets_size + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_offsets_size + row] = + static_cast(old_chars_size + (row + 1) * value_length); + } + sanity_check_simple(); + } + template void insert_many_strings_fixed_length(const StringRef* strings, size_t num) { size_t new_size = 0; diff --git a/be/src/core/column/column_variant.cpp b/be/src/core/column/column_variant.cpp index ec34c619cdc14b..aa910db72195b2 100644 --- a/be/src/core/column/column_variant.cpp +++ b/be/src/core/column/column_variant.cpp @@ -59,6 +59,7 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type/get_least_supertype.h" #include "core/data_type/primitive_type.h" +#include "core/data_type/storage_field_type.h" #include "core/field.h" #include "core/string_buffer.hpp" #include "core/types.h" @@ -1835,6 +1836,11 @@ bool ColumnVariant::is_visible_root_value(size_t nrow) const { } } + const auto& sparse_offsets = serialized_sparse_column_offsets(); + if (sparse_offsets[nrow - 1] != sparse_offsets[nrow]) { + return false; + } + const auto& doc_value_column_map = assert_cast(*serialized_doc_value_column); // doc snapshot column is not empty if (doc_value_column_map.get_offsets()[nrow - 1] != doc_value_column_map.get_offsets()[nrow]) { @@ -2746,7 +2752,7 @@ void ColumnVariant::Subcolumn::deserialize_from_binary_column(const ColumnString const auto& data_ref = value->get_data_at(row); const auto* start_data = reinterpret_cast(data_ref.data); const PrimitiveType type = - TabletColumn::get_primitive_type_by_field_type(static_cast(*start_data)); + storage_field_type_to_primitive_type(static_cast(*start_data)); auto check_end = [&](const uint8_t* end_ptr) { DCHECK_EQ(end_ptr - reinterpret_cast(data_ref.data), data_ref.size); }; diff --git a/be/src/core/data_type/data_type.cpp b/be/src/core/data_type/data_type.cpp index 94f7b6c38c9940..941a0377b05e66 100644 --- a/be/src/core/data_type/data_type.cpp +++ b/be/src/core/data_type/data_type.cpp @@ -32,9 +32,9 @@ #include "core/column/column.h" #include "core/column/column_const.h" #include "core/data_type/define_primitive_type.h" +#include "core/data_type/storage_field_type.h" #include "core/data_type_serde/data_type_serde.h" #include "core/field.h" -#include "storage/tablet/tablet_schema.h" namespace doris { class BufferWritable; @@ -47,7 +47,7 @@ IDataType::IDataType() = default; IDataType::~IDataType() = default; doris::FieldType IDataType::get_storage_field_type() const { - return TabletColumn::get_field_type_by_type(get_primitive_type()); + return primitive_type_to_storage_field_type(get_primitive_type()); } String IDataType::get_name() const { diff --git a/be/src/core/data_type/data_type.h b/be/src/core/data_type/data_type.h index 253bd8e49f99db..e1042bf4161a4c 100644 --- a/be/src/core/data_type/data_type.h +++ b/be/src/core/data_type/data_type.h @@ -38,6 +38,7 @@ #include "core/cow.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" +#include "core/data_type/storage_field_type.h" #include "core/data_type_serde/data_type_serde.h" #include "core/types.h" diff --git a/be/src/core/data_type/define_primitive_type.h b/be/src/core/data_type/define_primitive_type.h index b47218afc1bf98..852b68a96b5358 100644 --- a/be/src/core/data_type/define_primitive_type.h +++ b/be/src/core/data_type/define_primitive_type.h @@ -23,6 +23,9 @@ namespace doris { using PrimitiveNative = uint8_t; +// Compute-layer types. When adding a new value, decide how it maps to FieldType and explicitly +// define its behavior in primitive_type_to_storage_field_type() and +// storage_field_type_to_primitive_type(), either by providing a mapping or by throwing. enum PrimitiveType : PrimitiveNative { INVALID_TYPE = 0, TYPE_NULL, /* 1 */ diff --git a/be/src/core/data_type/storage_field_type.cpp b/be/src/core/data_type/storage_field_type.cpp new file mode 100644 index 00000000000000..269015401dcab7 --- /dev/null +++ b/be/src/core/data_type/storage_field_type.cpp @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/storage_field_type.h" + +#include "common/exception.h" +#include "storage/olap_common.h" + +namespace doris { + +// NOLINTNEXTLINE(readability-function-size): keep the exhaustive mapping together for auditability. +FieldType primitive_type_to_storage_field_type(PrimitiveType type) { + switch (type) { + case PrimitiveType::INVALID_TYPE: + return FieldType::OLAP_FIELD_TYPE_UNKNOWN; + case PrimitiveType::TYPE_NULL: + return FieldType::OLAP_FIELD_TYPE_NONE; + case PrimitiveType::TYPE_BOOLEAN: + return FieldType::OLAP_FIELD_TYPE_BOOL; + case PrimitiveType::TYPE_TINYINT: + return FieldType::OLAP_FIELD_TYPE_TINYINT; + case PrimitiveType::TYPE_SMALLINT: + return FieldType::OLAP_FIELD_TYPE_SMALLINT; + case PrimitiveType::TYPE_INT: + return FieldType::OLAP_FIELD_TYPE_INT; + case PrimitiveType::TYPE_BIGINT: + return FieldType::OLAP_FIELD_TYPE_BIGINT; + case PrimitiveType::TYPE_LARGEINT: + return FieldType::OLAP_FIELD_TYPE_LARGEINT; + case PrimitiveType::TYPE_FLOAT: + return FieldType::OLAP_FIELD_TYPE_FLOAT; + case PrimitiveType::TYPE_DOUBLE: + return FieldType::OLAP_FIELD_TYPE_DOUBLE; + case PrimitiveType::TYPE_VARCHAR: + return FieldType::OLAP_FIELD_TYPE_VARCHAR; + case PrimitiveType::TYPE_DATE: + return FieldType::OLAP_FIELD_TYPE_DATE; + case PrimitiveType::TYPE_DATETIME: + return FieldType::OLAP_FIELD_TYPE_DATETIME; + case PrimitiveType::TYPE_CHAR: + return FieldType::OLAP_FIELD_TYPE_CHAR; + case PrimitiveType::TYPE_STRUCT: + return FieldType::OLAP_FIELD_TYPE_STRUCT; + case PrimitiveType::TYPE_ARRAY: + return FieldType::OLAP_FIELD_TYPE_ARRAY; + case PrimitiveType::TYPE_MAP: + return FieldType::OLAP_FIELD_TYPE_MAP; + case PrimitiveType::TYPE_HLL: + return FieldType::OLAP_FIELD_TYPE_HLL; + case PrimitiveType::TYPE_DECIMALV2: + return FieldType::OLAP_FIELD_TYPE_DECIMAL; + case PrimitiveType::TYPE_BITMAP: + return FieldType::OLAP_FIELD_TYPE_BITMAP; + case PrimitiveType::TYPE_STRING: + return FieldType::OLAP_FIELD_TYPE_STRING; + case PrimitiveType::TYPE_QUANTILE_STATE: + return FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE; + case PrimitiveType::TYPE_DATEV2: + return FieldType::OLAP_FIELD_TYPE_DATEV2; + case PrimitiveType::TYPE_DATETIMEV2: + return FieldType::OLAP_FIELD_TYPE_DATETIMEV2; + case PrimitiveType::TYPE_TIMEV2: + return FieldType::OLAP_FIELD_TYPE_TIMEV2; + case PrimitiveType::TYPE_DECIMAL32: + return FieldType::OLAP_FIELD_TYPE_DECIMAL32; + case PrimitiveType::TYPE_DECIMAL64: + return FieldType::OLAP_FIELD_TYPE_DECIMAL64; + case PrimitiveType::TYPE_DECIMAL128I: + return FieldType::OLAP_FIELD_TYPE_DECIMAL128I; + case PrimitiveType::TYPE_JSONB: + return FieldType::OLAP_FIELD_TYPE_JSONB; + case PrimitiveType::TYPE_VARIANT: + return FieldType::OLAP_FIELD_TYPE_VARIANT; + case PrimitiveType::TYPE_AGG_STATE: + return FieldType::OLAP_FIELD_TYPE_AGG_STATE; + case PrimitiveType::TYPE_DECIMAL256: + return FieldType::OLAP_FIELD_TYPE_DECIMAL256; + case PrimitiveType::TYPE_IPV4: + return FieldType::OLAP_FIELD_TYPE_IPV4; + case PrimitiveType::TYPE_IPV6: + return FieldType::OLAP_FIELD_TYPE_IPV6; + case PrimitiveType::TYPE_UINT32: + return FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT; + case PrimitiveType::TYPE_UINT64: + return FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT; + case PrimitiveType::TYPE_TIMESTAMPTZ: + return FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ; + case PrimitiveType::TYPE_BINARY: + case static_cast(14): // TYPE_DECIMAL (deprecated) + case static_cast(21): // TYPE_TIME (deprecated) + case static_cast(33): // TYPE_LAMBDA_FUNCTION (deprecated) + case PrimitiveType::TYPE_FIXED_LENGTH_OBJECT: + case PrimitiveType::TYPE_VARBINARY: + break; + } + + throw Exception(ErrorCode::INTERNAL_ERROR, "Cannot convert PrimitiveType {} to FieldType", + static_cast(type)); +} + +// NOLINTNEXTLINE(readability-function-size): keep the exhaustive mapping together for auditability. +PrimitiveType storage_field_type_to_primitive_type(FieldType type) { + switch (type) { + case FieldType::OLAP_FIELD_TYPE_TINYINT: + return PrimitiveType::TYPE_TINYINT; + case FieldType::OLAP_FIELD_TYPE_SMALLINT: + return PrimitiveType::TYPE_SMALLINT; + case FieldType::OLAP_FIELD_TYPE_INT: + return PrimitiveType::TYPE_INT; + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT: + return PrimitiveType::TYPE_UINT32; + case FieldType::OLAP_FIELD_TYPE_BIGINT: + return PrimitiveType::TYPE_BIGINT; + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT: + return PrimitiveType::TYPE_UINT64; + case FieldType::OLAP_FIELD_TYPE_LARGEINT: + return PrimitiveType::TYPE_LARGEINT; + case FieldType::OLAP_FIELD_TYPE_FLOAT: + return PrimitiveType::TYPE_FLOAT; + case FieldType::OLAP_FIELD_TYPE_DOUBLE: + return PrimitiveType::TYPE_DOUBLE; + case FieldType::OLAP_FIELD_TYPE_CHAR: + return PrimitiveType::TYPE_CHAR; + case FieldType::OLAP_FIELD_TYPE_DATE: + return PrimitiveType::TYPE_DATE; + case FieldType::OLAP_FIELD_TYPE_DATETIME: + return PrimitiveType::TYPE_DATETIME; + case FieldType::OLAP_FIELD_TYPE_DECIMAL: + return PrimitiveType::TYPE_DECIMALV2; + case FieldType::OLAP_FIELD_TYPE_VARCHAR: + return PrimitiveType::TYPE_VARCHAR; + case FieldType::OLAP_FIELD_TYPE_STRUCT: + return PrimitiveType::TYPE_STRUCT; + case FieldType::OLAP_FIELD_TYPE_ARRAY: + return PrimitiveType::TYPE_ARRAY; + case FieldType::OLAP_FIELD_TYPE_MAP: + return PrimitiveType::TYPE_MAP; + case FieldType::OLAP_FIELD_TYPE_UNKNOWN: + return PrimitiveType::INVALID_TYPE; + case FieldType::OLAP_FIELD_TYPE_NONE: + return PrimitiveType::TYPE_NULL; + case FieldType::OLAP_FIELD_TYPE_HLL: + return PrimitiveType::TYPE_HLL; + case FieldType::OLAP_FIELD_TYPE_BOOL: + return PrimitiveType::TYPE_BOOLEAN; + case FieldType::OLAP_FIELD_TYPE_BITMAP: + return PrimitiveType::TYPE_BITMAP; + case FieldType::OLAP_FIELD_TYPE_STRING: + return PrimitiveType::TYPE_STRING; + case FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE: + return PrimitiveType::TYPE_QUANTILE_STATE; + case FieldType::OLAP_FIELD_TYPE_DATEV2: + return PrimitiveType::TYPE_DATEV2; + case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: + return PrimitiveType::TYPE_DATETIMEV2; + case FieldType::OLAP_FIELD_TYPE_TIMEV2: + return PrimitiveType::TYPE_TIMEV2; + case FieldType::OLAP_FIELD_TYPE_DECIMAL32: + return PrimitiveType::TYPE_DECIMAL32; + case FieldType::OLAP_FIELD_TYPE_DECIMAL64: + return PrimitiveType::TYPE_DECIMAL64; + case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: + return PrimitiveType::TYPE_DECIMAL128I; + case FieldType::OLAP_FIELD_TYPE_JSONB: + return PrimitiveType::TYPE_JSONB; + case FieldType::OLAP_FIELD_TYPE_VARIANT: + return PrimitiveType::TYPE_VARIANT; + case FieldType::OLAP_FIELD_TYPE_AGG_STATE: + return PrimitiveType::TYPE_AGG_STATE; + case FieldType::OLAP_FIELD_TYPE_DECIMAL256: + return PrimitiveType::TYPE_DECIMAL256; + case FieldType::OLAP_FIELD_TYPE_IPV4: + return PrimitiveType::TYPE_IPV4; + case FieldType::OLAP_FIELD_TYPE_IPV6: + return PrimitiveType::TYPE_IPV6; + case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: + return PrimitiveType::TYPE_TIMESTAMPTZ; + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT: + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT: + case FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE: + break; + } + + throw Exception(ErrorCode::INTERNAL_ERROR, "Cannot convert FieldType {} to PrimitiveType", + static_cast(type)); +} + +} // namespace doris diff --git a/be/src/core/data_type/storage_field_type.h b/be/src/core/data_type/storage_field_type.h new file mode 100644 index 00000000000000..a1d48e53eea2a5 --- /dev/null +++ b/be/src/core/data_type/storage_field_type.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "core/data_type/define_primitive_type.h" + +namespace doris { + +enum class FieldType; + +// Throws Exception when the primitive type has no storage field type mapping. +[[nodiscard]] FieldType primitive_type_to_storage_field_type(PrimitiveType type); + +// Throws Exception when the storage field type has no primitive type mapping. +[[nodiscard]] PrimitiveType storage_field_type_to_primitive_type(FieldType type); + +} // namespace doris diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 44402bb5b531cd..56a15feac494d8 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -31,10 +31,13 @@ #include "core/data_type/primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/data_type_serde/parquet_timestamp.h" #include "core/types.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" #include "exprs/function/cast/cast_to_string.h" +#include "util/unaligned.h" enum { DIVISOR_FOR_SECOND = 1, @@ -48,22 +51,6 @@ static const int64_t micro_to_nano_second = 1000; namespace { -#pragma pack(1) -struct DecodedInt96Timestamp { - int64_t nanos_of_day; - int32_t julian_day; - - int64_t to_timestamp_micros() const { - static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; - static constexpr int64_t MICROS_IN_DAY = 86400000000; - static constexpr int64_t NANOS_PER_MICROSECOND = 1000; - return (julian_day - JULIAN_EPOCH_OFFSET_DAYS) * MICROS_IN_DAY + - nanos_of_day / NANOS_PER_MICROSECOND; - } -}; -#pragma pack() -static_assert(sizeof(DecodedInt96Timestamp) == 12); - Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, int64_t timestamp_micros) { static constexpr int64_t MICROS_PER_SECOND = 1000000; @@ -105,9 +92,9 @@ Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, return Status::OK(); } -void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, - int64_t timestamp_micros, - const cctz::time_zone& timezone) { +Status append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, + int64_t timestamp_micros, + const cctz::time_zone& timezone) { static constexpr int64_t MICROS_PER_SECOND = 1000000; int64_t epoch_seconds = timestamp_micros / MICROS_PER_SECOND; @@ -120,19 +107,91 @@ void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, DateV2Value datetime_value; datetime_value.from_unixtime(epoch_seconds, timezone); datetime_value.set_microsecond(static_cast(micros_of_second)); + if (!datetime_value.is_valid_date()) { + return Status::DataQualityError( + "Decoded DATETIMEV2 timestamp is outside the target timezone range: micros={}", + timestamp_micros); + } data.push_back(datetime_value); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; + return ParquetTimeUnit::NANOS; } - return value; + return ParquetTimeUnit::MICROS; } +class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + DateTimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + static const auto utc_timezone = cctz::utc_time_zone(); + const auto& timezone = _context.timezone == nullptr ? utc_timezone : *_context.timezone; + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + Status status; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + status = _context.physical_type == ParquetPhysicalType::INT96 || + _context.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(_data, timestamp_micros); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectDateTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATETIMEV2"); + } +}; + } // namespace // NOLINTBEGIN(readability-function-size) @@ -525,16 +584,24 @@ Status DataTypeDateTimeV2SerDe::read_column_from_arrow(IColumn& column, for (auto value_i = start; value_i < end; ++value_i) { const uint8_t* raw_byte_ptr = base_ptr + value_i * element_size; auto date_value = unaligned_load(raw_byte_ptr); - auto utc_epoch = static_cast(date_value); DateV2Value v; - // convert second - v.from_unixtime(utc_epoch / divisor, ctz); - // get rest time + // C++ integer division truncates toward zero. Normalize the remainder so a negative + // timestamp still has a non-negative fractional part, e.g. -876544us becomes + // -1 second and 123456us. + int64_t seconds = date_value / divisor; + int64_t remainder = date_value % divisor; + if (remainder < 0) { + --seconds; + remainder += divisor; + } + v.from_unixtime(seconds, ctz); + // Get the fractional part. // add 0 on the right to make it 6 digits. DateTimeV2Value microsecond is 6 digits, // the scale decides to keep the first few digits, so the valid digits should be kept at the front. - // "2022-01-01 11:11:11.111", utc_epoch = 1641035471111, divisor = 1000, set_microsecond(111000) - v.set_microsecond((utc_epoch % divisor) * DIVISOR_FOR_MICRO / divisor); + // "2022-01-01 11:11:11.111", timestamp = 1641035471111, divisor = 1000, + // set_microsecond(111000) + v.set_microsecond(remainder * DIVISOR_FOR_MICRO / divisor); col_data.emplace_back(v); } } else { @@ -559,7 +626,7 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( auto& data = assert_cast(column).get_data(); const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); static const auto utc_timezone = cctz::utc_time_zone(); const auto& timezone = view.timezone == nullptr ? utc_timezone : *view.timezone; for (int64_t row = 0; row < view.row_count; ++row) { @@ -567,8 +634,19 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - append_datetimev2_from_utc_epoch_micros(data, values[row].to_timestamp_micros(), - timezone); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -581,24 +659,63 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - const int64_t timestamp_micros = decoded_timestamp_micros(view, values[row]); - if (view.timestamp_is_adjusted_to_utc) { - append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); - } else { - auto st = append_datetimev2_from_epoch_micros(data, timestamp_micros); - if (!st.ok()) { - if (decoded_column_view_can_null_on_conversion_failure(view)) { - decoded_column_view_insert_null_on_conversion_failure(column, view, row); - continue; - } - data.resize(old_size); - return st; + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = view.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; } + data.resize(old_size); + return status; } } return Status::OK(); } +Status DataTypeDateTimeV2SerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateTimeV2ParquetConsumer consumer(column, context); + RejectDateTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateTimeV2SerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("DATETIMEV2 expects Parquet INT64 or INT96"); + } + DateTimeV2ParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateTimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectDateTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + Status DataTypeDateTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.h b/be/src/core/data_type_serde/data_type_datetimev2_serde.h index e089d789ccee7d..c3411ed4249843 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.h @@ -90,6 +90,11 @@ class DataTypeDateTimeV2SerDe : public DataTypeNumberSerDe* value) { + DORIS_CHECK(value != nullptr); + const int64_t day_number = static_cast(encoded_date) + date_threshold; + if (day_number < 0 || !value->get_date_from_daynr(static_cast(day_number))) { + return Status::DataQualityError("Parquet DATE value is out of range"); + } + return Status::OK(); +} + +class DateV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + explicit DateV2ParquetConsumer(IColumn& column, ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + DateV2Value value; + const auto status = decode_parquet_date( + unaligned_load(values + row * sizeof(int32_t)), &value); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = DateV2Value(); + continue; + } + _data.resize(old_size); + return status; + } + _data[old_size + row] = value; + } + return Status::OK(); + } + +private: + ColumnDateV2::Container& _data; + ParquetMaterializationState* _state; +}; + +class RejectDateV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATEV2"); + } +}; + +} // namespace Status DataTypeDateV2SerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, BufferWritable& bw, @@ -140,6 +192,7 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::Corruption("Decoded value buffer is null for {}", column.get_name()); } auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); const auto* values = reinterpret_cast(view.values); for (int64_t row = 0; row < view.row_count; ++row) { if (decoded_column_view_row_is_null(view, row)) { @@ -147,12 +200,56 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column, continue; } DateV2Value date_v2; - date_v2.get_date_from_daynr(values[row] + date_threshold); + const auto status = decode_parquet_date(values[row], &date_v2); + if (!status.ok()) { + // Decoded values back both metadata conversion and native materialization. Preserve + // strict errors while allowing nullable non-strict scans to mark only the bad row. + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } data.push_back(date_v2); } return Status::OK(); } +Status DataTypeDateV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateV2ParquetConsumer consumer(column); + RejectDateV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT32 || + context.logical_type != ParquetLogicalType::DATE) { + return Status::NotSupported("DATEV2 expects Parquet DATE stored as INT32"); + } + DateV2ParquetConsumer consumer(column, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, &state); + RejectDateV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + Status DataTypeDateV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.h b/be/src/core/data_type_serde/data_type_datev2_serde.h index bc02b61b520193..39d44efcfa0652 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datev2_serde.h @@ -88,6 +88,11 @@ class DataTypeDateV2SerDe : public DataTypeNumberSerDe #include +#include #include #include @@ -33,14 +34,15 @@ #include "core/column/column_decimal.h" #include "core/data_type/data_type_decimal.h" #include "core/data_type/define_primitive_type.h" +#include "core/data_type/storage_field_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/types.h" #include "exec/common/arithmetic_overflow.h" #include "exprs/function/cast/cast_to_decimal.h" #include "exprs/function/cast/cast_to_string.h" #include "orc/Int128.hh" -#include "storage/tablet/tablet_schema.h" #include "util/jsonb_document.h" #include "util/jsonb_document_cast.h" #include "util/jsonb_writer.h" @@ -75,8 +77,13 @@ NativeType decode_big_endian_signed_integer(const uint8_t* data, int length) { template bool decoded_decimal_value_fits(const typename PrimitiveTypeTraits::CppType::NativeType& value, UInt32 precision) { - return value >= min_decimal_value(precision).value && - value <= max_decimal_value(precision).value; + if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; + } else { + return value >= min_decimal_value(precision).value && + value <= max_decimal_value(precision).value; + } } template @@ -85,6 +92,9 @@ bool decoded_decimal_int_value_fits(Int128 value, UInt32 precision) { if constexpr (std::is_same_v) { const auto wide_value = wide::Int256(value); return decoded_decimal_value_fits(wide_value, precision); + } else if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; } else { return value >= static_cast(min_decimal_value(precision).value) && value <= static_cast(max_decimal_value(precision).value); @@ -179,6 +189,214 @@ Status read_decimal_decoded_values(IColumn& column, const DecodedColumnView& vie return Status::OK(); } +template +wide::Int256 parquet_decimal_limit(UInt32 precision) { + if constexpr (T == TYPE_DECIMALV2) { + return wide::Int256(DataTypeDecimal::get_max_digits_number(precision)); + } else { + return wide::Int256(max_decimal_value(precision).value); + } +} + +template +Status scale_parquet_decimal(wide::Int256 value, int32_t source_scale, int32_t target_scale, + UInt32 target_precision, wide::Int256* result) { + DORIS_CHECK(result != nullptr); + const auto limit = parquet_decimal_limit(target_precision); + if (source_scale > target_scale) { + const int64_t scale_delta = static_cast(source_scale) - target_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // The precision bound above guarantees that narrowing the positive scale delta keeps + // the multiplier lookup in range. + const auto divisor = + decimal_scale_multiplier(static_cast(scale_delta)); + // Scale-down must be exact. Truncating before the target precision check silently + // changes source values and makes plain and dictionary decoding disagree with casts. + if (value % divisor != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + value /= divisor; + } + } else if (source_scale < target_scale) { + const int64_t scale_delta = static_cast(target_scale) - source_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // Keep the same checked narrowing invariant for scale-up as for exact scale-down. + const auto multiplier = + decimal_scale_multiplier(static_cast(scale_delta)); + if (value > limit / multiplier || value < -limit / multiplier) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + value *= multiplier; + } + } + if (value < -limit || value > limit) { + return Status::DataQualityError("Parquet decimal value is out of range"); + } + *result = value; + return Status::OK(); +} + +template +class DecimalParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + using FieldType = typename PrimitiveTypeTraits::CppType; + using NativeType = typename FieldType::NativeType; + + DecimalParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + UInt32 target_precision, int32_t target_scale, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast&>(column).get_data()), + _context(context), + _target_precision(target_precision), + _target_scale(target_scale), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_integers(values, num_values); + } + if (_context.physical_type == ParquetPhysicalType::INT64) { + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_integers(values, num_values); + } + if (_context.physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return Status::NotSupported("Unsupported Parquet physical type {} for decimal SerDe", + static_cast(_context.physical_type)); + } + DORIS_CHECK_EQ(value_width, static_cast(_context.type_length)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = + append_binary_value(values + row * value_width, value_width, old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = append_binary_value(reinterpret_cast(values[row].data), + values[row].size, old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const size_t length = value_offsets[row + 1] - value_offsets[row]; + auto status = append_binary_value( + reinterpret_cast(encoded_data + payload_offsets[row]), length, + old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + template + Status append_integers(const uint8_t* values, size_t num_values) { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + constexpr int32_t SOURCE_DIGITS = std::numeric_limits::digits10 + 1; + if (_context.decimal_scale == _target_scale && + SOURCE_DIGITS <= static_cast(_target_precision)) { + // The complete physical domain fits the target at the same scale, so narrowing cannot + // precede a failure check and the hot same-scale path needs no wide arithmetic. + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = + unaligned_load(values + row * sizeof(SourceType)); + _data[old_size + row] = FieldType {NativeType(source_value)}; + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = unaligned_load(values + row * sizeof(SourceType)); + auto status = append_wide_value(wide::Int256(source_value), old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status append_binary_value(const uint8_t* value, size_t length, size_t output_row) { + if (UNLIKELY(length > sizeof(wide::Int256))) { + return Status::DataQualityError("Parquet decimal binary value is too wide: {}", length); + } + return append_wide_value( + decode_big_endian_signed_integer(value, cast_set(length)), + output_row); + } + + Status append_wide_value(wide::Int256 value, size_t output_row) { + wide::Int256 scaled_value; + RETURN_IF_ERROR(scale_parquet_decimal(value, _context.decimal_scale, _target_scale, + _target_precision, &scaled_value)); + // Narrow only after scaling and target-precision validation. In particular, an INT64 or a + // sign-extended binary value must never wrap through Decimal32 before it can be rejected. + _data[output_row] = FieldType {static_cast(scaled_value)}; + return Status::OK(); + } + + typename ColumnDecimal::Container& _data; + const ParquetDecodeContext& _context; + UInt32 _target_precision; + int32_t _target_scale; + ParquetMaterializationState* _state; +}; + } // namespace template @@ -529,6 +747,49 @@ Status DataTypeDecimalSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeDecimalSerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeDecimalSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DecimalParquetConsumer dictionary_consumer(*state.typed_dictionary, context, + cast_set(precision), scale, &state); + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, dictionary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeDecimalSerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, @@ -853,7 +1114,7 @@ template void DataTypeDecimalSerDe::write_one_cell_to_binary(const IColumn& src_column, ColumnString::Chars& chars, int64_t row_num) const { - const uint8_t type = (const uint8_t)TabletColumn::get_field_type_by_type(T); + const uint8_t type = static_cast(primitive_type_to_storage_field_type(T)); const auto& data_ref = assert_cast&>(src_column).get_data_at(row_num); const auto& prec = static_cast(precision); const auto& sc = static_cast(scale); diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index 547488c9da4bc5..2a399ce83cc1a4 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -109,6 +109,11 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { int64_t end, const cctz::time_zone& ctz) const override; Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 3900f25a6d62c8..9474f39cb52c64 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -19,6 +19,8 @@ #include +#include +#include #include #include #include @@ -29,9 +31,11 @@ #include "core/column/column_nullable.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" +#include "core/data_type/storage_field_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/packed_int128.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -50,6 +54,23 @@ namespace doris { namespace { +float parquet_half_to_float(uint16_t half) { + const uint32_t sign = (half & 0x8000U) << 16; + const uint32_t exponent = (half & 0x7C00U) >> 10; + const uint32_t mantissa = half & 0x03FFU; + if (exponent == 0) { + if (mantissa == 0) { + return std::bit_cast(sign); + } + const float value = std::ldexp(static_cast(mantissa), -24); + return sign == 0 ? value : -value; + } + if (exponent == 0x1FU) { + return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); + } + return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); +} + template const NativeType* decoded_values_as(const DecodedColumnView& view) { return reinterpret_cast(view.values); @@ -77,6 +98,43 @@ bool decoded_number_value_fits(SourceType value) { } } +template +constexpr bool parquet_number_conversion_always_fits() { + if constexpr (std::is_floating_point_v) { + return true; + } else if constexpr (!std::is_integral_v || !std::is_integral_v || + std::is_same_v) { + return false; + } else if constexpr (std::is_signed_v == std::is_signed_v) { + return sizeof(DorisCppType) >= sizeof(SourceType); + } else if constexpr (std::is_signed_v) { + return sizeof(DorisCppType) > sizeof(SourceType); + } else { + return false; + } +} + +template +bool parquet_logical_integer_carrier_fits(SourceType value) { + if constexpr (sizeof(LogicalType) < sizeof(SourceType)) { + // Parquet INT(bitWidth) annotations constrain the physical INT32/INT64 carrier. Validate + // before narrowing so malformed values cannot wrap into an apparently valid logical value. + if constexpr (std::is_signed_v) { + const auto widened = static_cast(value); + if constexpr (std::is_signed_v) { + return widened >= static_cast(std::numeric_limits::lowest()) && + widened <= static_cast(std::numeric_limits::max()); + } + return widened >= 0 && + static_cast(widened) <= + static_cast(std::numeric_limits::max()); + } + const auto widened = static_cast(value); + return widened <= static_cast(std::numeric_limits::max()); + } + return true; +} + template Status read_number_decoded_values(IColumn& column, const DecodedColumnView& view) { if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { @@ -121,7 +179,21 @@ Status read_logical_integer_decoded_values_as(IColumn& column, const DecodedColu data.push_back(DorisCppType()); continue; } - const auto logical_value = static_cast(values[row]); + const auto physical_value = values[row]; + // Predicate decoding must match permissive materialization: annotated narrow integers use + // their declared bit width unless strict metadata validation explicitly requests rejection. + if (view.enable_strict_mode && + !parquet_logical_integer_carrier_fits(physical_value)) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return Status::DataQualityError( + "Decoded logical integer carrier is out of range for {} at row {}", + column.get_name(), row); + } + const auto logical_value = static_cast(physical_value); if (!decoded_number_value_fits(logical_value)) { if (decoded_column_view_can_null_on_conversion_failure(view)) { decoded_column_view_insert_null_on_conversion_failure(column, view, row); @@ -178,6 +250,226 @@ Status read_integer_decoded_values(IColumn& column, const DecodedColumnView& vie } } +template +Status append_parquet_number(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + if constexpr (std::is_same_v) { + // Identical fixed-width physical/logical types need no validation or conversion. Parquet + // PLAIN values and Doris POD columns share the byte representation on supported targets, + // so preserve one dense vector-at-a-time memcpy instead of converting value by value. + memcpy(data.data() + old_size, values, num_values * sizeof(SourceType)); + return Status::OK(); + } + if constexpr (parquet_number_conversion_always_fits()) { + // A widening conversion cannot fail, so keeping range checks in the row loop only blocks + // auto-vectorization. Input can be unaligned at a Parquet page boundary; load explicitly. + for (size_t row = 0; row < num_values; ++row) { + data[old_size + row] = static_cast( + unaligned_load(values + row * sizeof(SourceType))); + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load(values + row * sizeof(SourceType)); + if (!decoded_number_value_fits(value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError("Parquet value is out of range at row {}", row); + } + data[old_size + row] = static_cast(value); + } + return Status::OK(); +} + +template +Status append_parquet_logical_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values, ParquetMaterializationState* state) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + if constexpr (parquet_number_conversion_always_fits() && + sizeof(LogicalType) >= sizeof(SourceType)) { + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = + unaligned_load(values + row * sizeof(SourceType)); + data[old_size + row] = + static_cast(static_cast(physical_value)); + } + return Status::OK(); + } + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = unaligned_load(values + row * sizeof(SourceType)); + // Permissive scans preserve the long-standing bit-width interpretation of annotated + // carriers; strict scans still reject malformed physical values before narrowing. + if ((state == nullptr || state->enable_strict_mode) && + !parquet_logical_integer_carrier_fits(physical_value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError( + "Parquet logical integer carrier is out of range at row {}", row); + } + const auto logical_value = static_cast(physical_value); + if (!decoded_number_value_fits(logical_value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); + continue; + } + data.resize(old_size); + return Status::DataQualityError("Parquet logical integer is out of range at row {}", + row); + } + data[old_size + row] = static_cast(logical_value); + } + return Status::OK(); +} + +template +Status append_parquet_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { + if (context.logical_integer_bit_width <= 0) { + return append_parquet_number(data, values, num_values, context, + state); + } + if (context.logical_integer_is_signed) { + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers( + data, values, num_values, state); + case 16: + return append_parquet_logical_integers( + data, values, num_values, state); + case 32: + return append_parquet_logical_integers( + data, values, num_values, state); + case 64: + return append_parquet_logical_integers( + data, values, num_values, state); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } + } + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers(data, values, + num_values, state); + case 16: + return append_parquet_logical_integers(data, values, + num_values, state); + case 32: + return append_parquet_logical_integers(data, values, + num_values, state); + case 64: + return append_parquet_logical_integers(data, values, + num_values, state); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } +} + +template +class NumberParquetConsumer final : public ParquetFixedValueConsumer { +public: + using DorisCppType = typename PrimitiveTypeTraits::CppType; + using ColumnType = typename PrimitiveTypeTraits::ColumnType; + + NumberParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return consume_impl(values, num_values, value_width); + } + + Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) override { + // Each selected PLAIN range is already contiguous in the encoded page. Append those spans + // directly so sparse reads never build an intermediate selected-values array. + for (const auto& range : ranges) { + RETURN_IF_ERROR( + consume_impl(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } + +private: + Status consume_impl(const uint8_t* values, size_t num_values, size_t value_width) { + if (_context.logical_float16) { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY); + DORIS_CHECK_EQ(value_width, sizeof(uint16_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const float value = parquet_half_to_float( + unaligned_load(values + row * sizeof(uint16_t))); + if (!decoded_number_value_fits(value)) { + if (_state != nullptr && _state->can_insert_null_on_conversion_failure()) { + _data[old_size + row] = DorisCppType(); + DORIS_CHECK(_state->mark_conversion_failure(old_size + row)); + continue; + } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet FLOAT16 value is out of range at row {}", row); + } + _data[old_size + row] = static_cast(value); + } + return Status::OK(); + } + switch (_context.physical_type) { + case ParquetPhysicalType::BOOLEAN: + DORIS_CHECK_EQ(value_width, sizeof(uint8_t)); + return append_parquet_number(_data, values, num_values, _context, + _state); + case ParquetPhysicalType::INT32: + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_parquet_integers(_data, values, num_values, + _context, _state); + case ParquetPhysicalType::INT64: + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_parquet_integers(_data, values, num_values, + _context, _state); + case ParquetPhysicalType::FLOAT: + DORIS_CHECK_EQ(value_width, sizeof(float)); + return append_parquet_number(_data, values, num_values, _context, + _state); + case ParquetPhysicalType::DOUBLE: + DORIS_CHECK_EQ(value_width, sizeof(double)); + return append_parquet_number(_data, values, num_values, _context, + _state); + default: + return Status::NotSupported("Unsupported Parquet physical type {} for numeric SerDe", + static_cast(_context.physical_type)); + } + } + + PaddedPODArray& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectParquetBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as a number"); + } +}; + } // namespace // Basic structure of the type map. template @@ -329,6 +621,52 @@ Status DataTypeNumberSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeNumberSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_parquet_dictionary(column, source, context); + } else { + NumberParquetConsumer consumer(column, context); + RejectParquetBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); + } +} + +template +Status DataTypeNumberSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_column_from_parquet(column, source, context, num_values, state); + } else { + NumberParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + NumberParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectParquetBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); + } +} + template Status DataTypeNumberSerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const { @@ -978,8 +1316,8 @@ Status DataTypeNumberSerDe::from_string(StringRef& str, IColumn& column, // Format by type: // - BOOLEAN: "0" or "1" (via snprintf "%d") // - TINYINT/SMALLINT/INT/BIGINT: standard integer string, e.g. "42", "-100" -// - FLOAT: fmt::format("{:.7g}", value), e.g. "3.14", "NaN", "Infinity" -// - DOUBLE: fmt::format("{:.16g}", value), e.g. "3.141592653589793" +// - FLOAT: fmt::format("{}", value) for finite values, e.g. "3.14" +// - DOUBLE: fmt::format("{}", value) for finite values, e.g. "3.141592653589793" // - LARGEINT: fmt::format("{}", value), e.g. "170141183460469231731687303715884105727" // // Examples: @@ -992,8 +1330,18 @@ std::string DataTypeNumberSerDe::to_olap_string(const Field& field) const { char buf[8] = {'\0'}; snprintf(buf, sizeof(buf), "%d", field.get()); return std::string(buf); + } else if constexpr (T == TYPE_FLOAT || T == TYPE_DOUBLE) { + auto v = field.get(); + // inf/nan are stored as zone-map flags, not in min/max strings; route them + // through CastToString to keep the "Infinity"/"NaN" spelling used elsewhere. + if (std::isinf(v) || std::isnan(v)) { + return CastToString::from_number(v); + } + // CastToString uses digits10 + 1 significant digits, which may lose precision. + // ZoneMap bounds must round-trip exactly, so use fmt's shortest round-trippable form. + return fmt::format("{}", v); } else if constexpr (T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || - T == TYPE_BIGINT || T == TYPE_FLOAT || T == TYPE_DOUBLE) { + T == TYPE_BIGINT) { return CastToString::from_number(field.get()); } else if constexpr (T == TYPE_LARGEINT) { auto value = field.get(); @@ -1114,7 +1462,7 @@ template void DataTypeNumberSerDe::write_one_cell_to_binary(const IColumn& src_column, ColumnString::Chars& chars, int64_t row_num) const { - const uint8_t type = (const uint8_t)TabletColumn::get_field_type_by_type(T); + const uint8_t type = static_cast(primitive_type_to_storage_field_type(T)); const auto& data_ref = assert_cast(src_column).get_data_at(row_num); const size_t old_size = chars.size(); diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 4ced14ed858f94..49196d3956d01e 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -119,6 +119,11 @@ class DataTypeNumberSerDe : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index 16526bc172b213..c84203b6d7f47b 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -42,6 +42,7 @@ #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/storage_field_type.h" #include "core/data_type_serde/data_type_array_serde.h" #include "core/data_type_serde/data_type_datetimev2_serde.h" #include "core/data_type_serde/data_type_datev2_serde.h" @@ -51,12 +52,14 @@ #include "core/data_type_serde/data_type_number_serde.h" #include "core/data_type_serde/data_type_string_serde.h" #include "core/data_type_serde/data_type_timestamptz_serde.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/field.h" #include "core/types.h" #include "core/value/timestamptz_value.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_base.h" #include "runtime/descriptors.h" +#include "storage/olap_common.h" #include "util/jsonb_document.h" #include "util/jsonb_writer.h" namespace doris { @@ -708,6 +711,18 @@ Status DataTypeSerDe::read_column_from_decoded_values(IColumn& column, get_name())); } +Status DataTypeSerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + return Status::NotSupported("read_column_from_parquet is not supported for {}", get_name()); +} + +Status DataTypeSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + return Status::NotSupported("read_parquet_dictionary is not supported for {}", get_name()); +} + Status DataTypeSerDe::read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const { return Status::NotSupported("read_column_from_orc is not supported for {}", get_name()); @@ -1130,7 +1145,7 @@ const uint8_t* DataTypeSerDe::deserialize_binary_to_column(const uint8_t* data, const uint8_t* DataTypeSerDe::deserialize_binary_to_field(const uint8_t* data, Field& field, FieldInfo& info) { const FieldType type = static_cast(*data++); - info.scalar_type_id = TabletColumn::get_primitive_type_by_field_type(type); + info.scalar_type_id = storage_field_type_to_primitive_type(type); const uint8_t* end = data; switch (type) { #define HANDLE_SIMPLE_SERDE(FT, SERDE) \ diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index edeb0b0de28781..49bfb73cbd854c 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -93,6 +93,9 @@ struct CastParameters; class DataTypeSerDe; using DataTypeSerDeSPtr = std::shared_ptr; using DataTypeSerDeSPtrs = std::vector; +class ParquetDecodeSource; +struct ParquetDecodeContext; +struct ParquetMaterializationState; /// Info that represents a scalar or array field in a decomposed view. /// It allows to recreate field with different number @@ -505,6 +508,18 @@ class DataTypeSerDe { // the Doris-type-specific materialization into IColumn. virtual Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const; + + // Read encoded Parquet values directly into the destination Doris column. ColumnReader owns + // levels/null/filter handling; the source owns only encoding-stream state; the target SerDe + // owns all physical/logical type interpretation and materialization. + virtual Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const; + // Decode one dictionary page into the selected Doris type without consuming data-page + // indices. Dictionary filters use this to keep type interpretation in SerDe instead of + // exposing dictionary bytes or decoder-owned strings to ColumnReader. + virtual Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const; virtual Status read_field_from_decoded_value(const IDataType& data_type, Field* field, const DecodedColumnView& view) const; diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index bd1c6e5b4f19e2..24f7215d42e044 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -17,14 +17,18 @@ #include "core/data_type_serde/data_type_string_serde.h" +#include #include #include +#include #include "common/config.h" #include "core/column/column_string.h" +#include "core/column/column_vector.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "util/jsonb_document_cast.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" @@ -57,6 +61,65 @@ Status read_string_decoded_values(IColumn& column, const DecodedColumnView& view return Status::OK(); } +template +class StringParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + explicit StringParquetConsumer(IColumn& column) : _column(assert_cast(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if constexpr (requires(ColumnType& column) { + column.insert_many_fixed_length_data(static_cast(nullptr), + size_t(), size_t()); + }) { + // FIXED_LEN_BYTE_ARRAY is already a dense byte span. Copy it once and synthesize + // offsets; StringRef batches add a second row loop and hundreds of tiny memcpy calls. + _column.insert_many_fixed_length_data(reinterpret_cast(values), + value_width, num_values); + } else { + static constexpr size_t BATCH_SIZE = 256; + std::array refs; + size_t offset = 0; + while (offset < num_values) { + const size_t batch_size = std::min(BATCH_SIZE, num_values - offset); + for (size_t row = 0; row < batch_size; ++row) { + refs[row] = StringRef( + reinterpret_cast(values + (offset + row) * value_width), + value_width); + } + _column.insert_many_strings(refs.data(), batch_size); + offset += batch_size; + } + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + _column.insert_many_strings(values, num_values); + return Status::OK(); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + if constexpr (requires(ColumnType& column) { + column.insert_many_parquet_plain_byte_arrays( + encoded_data, payload_offsets, value_offsets, num_values, + value_spans); + }) { + _column.insert_many_parquet_plain_byte_arrays(encoded_data, payload_offsets, + value_offsets, num_values, value_spans); + return Status::OK(); + } + return ParquetBinaryValueConsumer::consume_plain_byte_array( + encoded_data, payload_offsets, value_offsets, num_values, value_spans); + } + +private: + ColumnType& _column; +}; + } // namespace namespace { @@ -496,6 +559,57 @@ Status DataTypeStringSerDeBase::read_column_from_decoded_values( return read_string_decoded_values(column, view); } +template +Status DataTypeStringSerDeBase::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + StringParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeStringSerDeBase::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.dictionary_index_only) { + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return Status::IOError("Dictionary filter requested for a non-dictionary page"); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + auto& indices = assert_cast(column).get_data(); + const size_t old_size = indices.size(); + indices.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(state.dictionary_indices[row] > + static_cast(std::numeric_limits::max()))) { + indices.resize(old_size); + return Status::Corruption("Parquet dictionary index {} exceeds INT32", + state.dictionary_indices[row]); + } + indices[old_size + row] = static_cast(state.dictionary_indices[row]); + } + return Status::OK(); + } + StringParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type {} for string SerDe", + static_cast(context.physical_type)); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeStringSerDeBase::write_column_to_orc( const std::string& timezone, const IColumn& column, const NullMap* null_map, diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index ad9a05dc996358..c81f6d37aa4ffc 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -205,6 +205,11 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index c40e671793c848..52d387eda5ca6d 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -17,13 +17,17 @@ #include "core/data_type_serde/data_type_time_serde.h" +#include + #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/value/time_value.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_time_impl.hpp" +#include "util/unaligned.h" namespace doris { namespace { @@ -51,6 +55,74 @@ TimeValue::TimeType read_time_decoded_value(const DecodedColumnView& view, int64 abs_micros % TimeValue::ONE_SECOND_MICROSECONDS, negative); } +class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + int64_t raw_value; + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + raw_value = unaligned_load(values + row * sizeof(int32_t)); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + raw_value = unaligned_load(values + row * sizeof(int64_t)); + } + + int64_t units_per_day; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + units_per_day = 86400000; + } else if (_context.time_unit == ParquetTimeUnit::MICROS) { + units_per_day = 86400000000; + } else { + DORIS_CHECK(_context.time_unit == ParquetTimeUnit::NANOS); + units_per_day = 86400000000000; + } + // Validate the declared carrier before rescaling: truncating nanoseconds first could + // turn a value at or beyond 24:00:00 into an apparently valid TIMEV2 value. + if (raw_value < 0 || raw_value >= units_per_day) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = TimeValue::TimeType(); + continue; + } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet TIME value {} is outside the one-day domain", raw_value); + } + int64_t micros = raw_value; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + micros *= 1000; + } else if (_context.time_unit == ParquetTimeUnit::NANOS) { + micros /= 1000; + } + // Doris TIMEV2 stores signed microseconds in a double. Splitting into calendar fields + // and immediately recombining them is an identity operation with several divisions. + _data[old_size + row] = static_cast(micros); + } + return Status::OK(); + } + +private: + ColumnTimeV2::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMEV2"); + } +}; + } // namespace Status DataTypeTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, @@ -193,6 +265,41 @@ Status DataTypeTimeV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::OK(); } +Status DataTypeTimeV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + TimeV2ParquetConsumer consumer(column, context); + RejectTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if ((context.physical_type != ParquetPhysicalType::INT32 && + context.physical_type != ParquetPhysicalType::INT64) || + context.logical_type != ParquetLogicalType::TIME) { + return Status::NotSupported("TIMEV2 expects Parquet TIME stored as INT32 or INT64"); + } + TimeV2ParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + template Status DataTypeTimeV2SerDe::from_int_batch(const typename IntDataType::ColumnType& int_col, ColumnNullable& target_col) const { diff --git a/be/src/core/data_type_serde/data_type_time_serde.h b/be/src/core/data_type_serde/data_type_time_serde.h index e3fccf379c913a..20fa14a148854a 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.h +++ b/be/src/core/data_type_serde/data_type_time_serde.h @@ -69,6 +69,11 @@ class DataTypeTimeV2SerDe : public DataTypeNumberSerDe(micros_of_second)); + if (!timestamp_tz.is_valid_date()) { + return Status::DataQualityError( + "Decoded TIMESTAMPTZ is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } data.push_back(timestamp_tz); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; + return ParquetTimeUnit::NANOS; } - return value; + return ParquetTimeUnit::MICROS; } +class TimestampTzParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimestampTzParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + Status status; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + } + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(_data, timestamp_micros); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnTimeStampTz::Container& _data; + const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; +}; + +class RejectTimestampTzBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMESTAMPTZ"); + } +}; + } // namespace // The implementation of these functions mainly refers to data_type_datetimev2_serde.cpp @@ -308,14 +355,27 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( } auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); for (int64_t row = 0; row < view.row_count; ++row) { if (decoded_column_view_row_is_null(view, row)) { data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, values[row].to_timestamp_micros()); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -326,11 +386,57 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, decoded_timestamp_micros(view, values[row])); + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } +Status DataTypeTimeStampTzSerDe::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + TimestampTzParquetConsumer consumer(column, context); + RejectTimestampTzBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeStampTzSerDe::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("TIMESTAMPTZ expects Parquet INT64 or INT96"); + } + TimestampTzParquetConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimestampTzParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimestampTzBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} + std::string DataTypeTimeStampTzSerDe::to_olap_string(const Field& field) const { return CastToString::from_timestamptz(field.get(), 6); } diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.h b/be/src/core/data_type_serde/data_type_timestamptz_serde.h index 23d57f57fc8dac..6777459909a090 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.h +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.h @@ -75,6 +75,11 @@ class DataTypeTimeStampTzSerDe : public DataTypeNumberSerDe(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(reinterpret_cast(values + row * value_width), + value_width); + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(values[row].data, values[row].size); + } + return Status::OK(); + } + + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return Status::OK(); + } + +private: + ColumnVarbinary& _column; +}; + +} // namespace + +Status DataTypeVarbinarySerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + VarbinaryParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +Status DataTypeVarbinarySerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + VarbinaryParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type for VARBINARY"); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + return state.materialize_dictionary(column, source, num_values); +} void DataTypeVarbinarySerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena& arena, int32_t col_id, int64_t row_num, diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.h b/be/src/core/data_type_serde/data_type_varbinary_serde.h index 41dfa3fce002f8..6d23159c8ea5fe 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.h +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.h @@ -81,6 +81,12 @@ class DataTypeVarbinarySerDe : public DataTypeSerDe { "read_column_from_arrow with type " + column.get_name()); } + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, const FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h new file mode 100644 index 00000000000000..e9762385689d07 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/check.h" +#include "common/status.h" +#include "core/column/column.h" +#include "core/string_ref.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris { + +// These enums deliberately do not expose parquet thrift classes to the core type system. The +// format reader translates the thrift metadata once when it creates a column reader. +enum class ParquetPhysicalType { + BOOLEAN, + INT32, + INT64, + INT96, + FLOAT, + DOUBLE, + BYTE_ARRAY, + FIXED_LEN_BYTE_ARRAY, +}; + +enum class ParquetValueEncoding { + PLAIN, + DICTIONARY, + RLE, + BIT_PACKED, + DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, + BYTE_STREAM_SPLIT, +}; + +enum class ParquetTimeUnit { + UNKNOWN, + MILLIS, + MICROS, + NANOS, +}; + +enum class ParquetLogicalType { + NONE, + STRING, + DECIMAL, + DATE, + TIME, + TIMESTAMP, + INTEGER, + UUID, + FLOAT16, +}; + +// Immutable metadata required to turn one Parquet physical value into the selected Doris type. +// Encoding describes how the value source is read; logical annotations describe its meaning. +struct ParquetDecodeContext { + ParquetPhysicalType physical_type = ParquetPhysicalType::INT32; + ParquetValueEncoding encoding = ParquetValueEncoding::PLAIN; + ParquetLogicalType logical_type = ParquetLogicalType::NONE; + ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; + + int32_t type_length = -1; + int32_t decimal_precision = -1; + int32_t decimal_scale = -1; + int32_t logical_integer_bit_width = -1; + bool logical_integer_is_signed = true; + bool timestamp_is_adjusted_to_utc = false; + bool logical_float16 = false; + bool logical_uuid = false; + bool dictionary_index_only = false; + + const cctz::time_zone* timezone = nullptr; +}; + +struct ParquetSelectionRange { + size_t first = 0; + size_t count = 0; +}; + +// A decoder may produce multiple contiguous spans for one request (for example delta encodings). +// Consumers are invoked per span, never per value, keeping virtual dispatch out of the row loop. +class ParquetFixedValueConsumer { +public: + virtual ~ParquetFixedValueConsumer() = default; + virtual Status consume(const uint8_t* values, size_t num_values, size_t value_width) = 0; + virtual Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) { + for (const auto& range : ranges) { + RETURN_IF_ERROR(consume(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } +}; + +class ParquetBinaryValueConsumer { +public: + virtual ~ParquetBinaryValueConsumer() = default; + virtual Status consume(const StringRef* values, size_t num_values) = 0; + + // PLAIN BYTE_ARRAY decoders already have to parse every length prefix. Publish the parsed + // source and destination offsets so string columns do not rebuild an equally large StringRef + // array and rescan all lengths before copying. Spans are expressed in output coordinates and + // preserve adjacent surviving runs for consumers that can amortize range setup. + virtual Status consume_plain_byte_array(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) { + std::vector values; + values.reserve(num_values); + for (size_t row = 0; row < num_values; ++row) { + values.emplace_back(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return consume(values.data(), values.size()); + } +}; + +// Dictionary decoders publish validated IDs without knowing the destination Doris type. A +// cache-resident dictionary can therefore fuse RLE decode with target-column gathering, while a +// large sparse dictionary can still choose a compact ID buffer before random dictionary access. +class ParquetDictionaryValueConsumer { +public: + virtual ~ParquetDictionaryValueConsumer() = default; + virtual Status consume_indices(const uint32_t* indices, size_t num_values) = 0; + virtual Status consume_repeated(uint32_t index, size_t num_values) { + std::vector indices(num_values, index); + return consume_indices(indices.data(), indices.size()); + } +}; + +// Physical value ranges selected from one page-bounded decode request. Definition-level NULLs do +// not occupy the encoded value stream, so the native ColumnReader merges the logical selection +// with definition levels before constructing this plan. The SerDe sees only selected non-NULL +// payload, while the reader restores selected NULL slots after the compact decode. Ranges are +// sorted, disjoint, and expressed in the physical value stream's coordinate space. +struct ParquetSelection { + size_t total_values = 0; + size_t selected_values = 0; + std::vector ranges; +}; + +// Encoding decoders implement this interface. They own encoded-stream cursors and dictionary +// storage, but they never know the destination Doris column type. DataTypeSerDe owns the consumer +// and therefore the physical/logical-to-Doris conversion. +class ParquetDecodeSource { +public: + virtual ~ParquetDecodeSource() = default; + + virtual Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) = 0; + virtual Status decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) = 0; + virtual Status skip_values(size_t num_values) = 0; + + // Batch-level sparse decode. The default implementation preserves every encoding's cursor + // semantics while moving SerDe dispatch and consumer construction out of the selection-run + // loop. Decoders with cheap random access or batch decode override these methods to remove the + // remaining per-range virtual calls as well. + virtual Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_fixed_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + virtual Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_binary_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + + virtual bool has_dictionary() const { return false; } + virtual uint64_t dictionary_generation() const { return 0; } + virtual size_t dictionary_size() const { return 0; } + virtual Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return Status::NotSupported("Parquet dictionary is not supported by this decoder"); + } + virtual Status decode_dictionary_indices(size_t num_values, std::vector* indices) { + return Status::NotSupported("Parquet dictionary indices are not supported by this decoder"); + } + virtual Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) { + DORIS_CHECK(indices != nullptr); + indices->clear(); + indices->reserve(selection.selected_values); + std::vector range_indices; + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_dictionary_indices(range.count, &range_indices)); + indices->insert(indices->end(), range_indices.begin(), range_indices.end()); + cursor = range.first + range.count; + } + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + DORIS_CHECK_EQ(indices->size(), selection.selected_values); + return Status::OK(); + } + virtual Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_dictionary_indices(num_values, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_selected_dictionary_indices(selection, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const { + return false; + } +}; + +enum class ParquetDictionaryMaterializationStrategy : uint8_t { DIRECT, INDICES }; + +// Dictionary values are materialized once into the selected Doris type. The state belongs to a +// column reader rather than DataTypeSerDe because a SerDe instance can be shared by many files. +struct ParquetMaterializationState { + MutableColumnPtr typed_dictionary; + std::vector dictionary_indices; + ParquetSelection selection; + uint64_t dictionary_generation = std::numeric_limits::max(); + bool enable_strict_mode = false; + IColumn::Filter* conversion_failure_null_map = nullptr; + IColumn::Filter dictionary_conversion_failures; + bool capturing_dictionary_conversion_failures = false; + bool dictionary_has_conversion_failures = false; + size_t dictionary_failure_scan_rows = 0; + ParquetDictionaryMaterializationStrategy dictionary_materialization_strategy = + ParquetDictionaryMaterializationStrategy::INDICES; + + void reset_dictionary() { + typed_dictionary.reset(); + dictionary_indices.clear(); + dictionary_conversion_failures.clear(); + capturing_dictionary_conversion_failures = false; + dictionary_has_conversion_failures = false; + dictionary_failure_scan_rows = 0; + dictionary_generation = std::numeric_limits::max(); + } + + bool can_insert_null_on_conversion_failure() const { + return conversion_failure_null_map != nullptr && + (!enable_strict_mode || capturing_dictionary_conversion_failures); + } + + bool mark_conversion_failure(size_t output_row) { + if (!can_insert_null_on_conversion_failure()) { + return false; + } + DORIS_CHECK_LT(output_row, conversion_failure_null_map->size()); + (*conversion_failure_null_map)[output_row] = 1; + if (capturing_dictionary_conversion_failures) { + dictionary_has_conversion_failures = true; + } + return true; + } + + IColumn::Filter* begin_dictionary_conversion(size_t dictionary_size) { + auto* output_null_map = conversion_failure_null_map; + dictionary_conversion_failures.resize_fill(dictionary_size, 0); + dictionary_has_conversion_failures = false; + conversion_failure_null_map = &dictionary_conversion_failures; + capturing_dictionary_conversion_failures = true; + return output_null_map; + } + + void end_dictionary_conversion(IColumn::Filter* output_null_map) { + conversion_failure_null_map = output_null_map; + capturing_dictionary_conversion_failures = false; + } + + Status materialize_dictionary(IColumn& column) { + const size_t old_size = column.size(); + dictionary_failure_scan_rows = 0; + if (UNLIKELY(dictionary_has_conversion_failures)) { + const bool insert_failure_as_null = can_insert_null_on_conversion_failure(); + if (!insert_failure_as_null) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] == 0) { + continue; + } + // A malformed dictionary entry is irrelevant until a selected row references + // it; failing while building the dictionary would reject valid pages. + return Status::DataQualityError( + "Parquet dictionary entry {} cannot be converted to the target type", + dictionary_id); + } + } + } + column.insert_indices_from(*typed_dictionary, dictionary_indices.data(), + dictionary_indices.data() + dictionary_indices.size()); + if (UNLIKELY(dictionary_has_conversion_failures && + can_insert_null_on_conversion_failure())) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] != 0) { + mark_conversion_failure(old_size + row); + } + } + } + return Status::OK(); + } + + Status materialize_dictionary(IColumn& column, ParquetDecodeSource& source, size_t num_values) { + DORIS_CHECK(typed_dictionary); + if (UNLIKELY(dictionary_has_conversion_failures) || + source.prefer_dictionary_index_materialization(typed_dictionary->byte_size())) { + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::INDICES; + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &dictionary_indices)); + DORIS_CHECK_EQ(dictionary_indices.size(), num_values); + return materialize_dictionary(column); + } + + class ColumnConsumer final : public ParquetDictionaryValueConsumer { + public: + ColumnConsumer(IColumn& destination, const IColumn& dictionary) + : _destination(destination), _dictionary(dictionary) {} + + Status consume_indices(const uint32_t* indices, size_t num_values) override { + _destination.insert_indices_from(_dictionary, indices, indices + num_values); + return Status::OK(); + } + + Status consume_repeated(uint32_t index, size_t num_values) override { + _destination.insert_many_from(_dictionary, index, num_values); + return Status::OK(); + } + + private: + IColumn& _destination; + const IColumn& _dictionary; + } consumer(column, *typed_dictionary); + + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::DIRECT; + const size_t old_size = column.size(); + const Status status = source.decode_dictionary_values(num_values, consumer); + if (!status.ok()) { + // Streaming direct gather may discover a corrupt late ID after earlier valid runs; + // restore the same all-or-nothing column invariant as the index-buffer path. + column.resize(old_size); + } + return status; + } +}; + +} // namespace doris diff --git a/be/src/core/data_type_serde/parquet_timestamp.h b/be/src/core/data_type_serde/parquet_timestamp.h new file mode 100644 index 00000000000000..ba2aa686ad4272 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_timestamp.h @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { + +#pragma pack(1) +struct ParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(ParquetInt96Timestamp) == 12); + +inline constexpr int64_t MIN_DORIS_TIMESTAMP_MICROS = -62135596800000000LL; +inline constexpr int64_t MAX_DORIS_TIMESTAMP_MICROS = 253402300799999999LL; + +inline Status validate_parquet_timestamp_micros(int64_t timestamp_micros) { + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet timestamp is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } + return Status::OK(); +} + +inline Status parquet_timestamp_micros(ParquetTimeUnit unit, int64_t value, int64_t* result) { + if (unit == ParquetTimeUnit::MILLIS) { + // Validate in the source unit before scaling; signed overflow could otherwise turn an + // invalid file value into an in-range timestamp that survives later range checks. + if (value > std::numeric_limits::max() / 1000 || + value < std::numeric_limits::min() / 1000) { + return Status::DataQualityError("Parquet timestamp overflows microseconds"); + } + *result = value * 1000; + } else if (unit == ParquetTimeUnit::NANOS) { + *result = value / 1000; + // Epoch-relative negative timestamps must round toward the preceding representable + // microsecond; C++ division truncates toward zero and would move pre-epoch values forward. + if (value < 0 && value % 1000 != 0) { + --*result; + } + } else { + *result = value; + } + return validate_parquet_timestamp_micros(*result); +} + +inline Status parquet_int96_timestamp_micros(const ParquetInt96Timestamp& value, int64_t* result) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_IN_DAY = 86400000000LL; + static constexpr int64_t NANOS_IN_DAY = 86400000000000LL; + static constexpr int64_t NANOS_PER_MICROSECOND = 1000; + + // INT96 nanos is a time-of-day, not a signed duration. Validate it before widened day + // arithmetic so corrupt input cannot wrap into a plausible Doris timestamp. + if (value.nanos_of_day < 0 || value.nanos_of_day >= NANOS_IN_DAY) { + return Status::DataQualityError("Invalid Parquet INT96 nanos-of-day: {}", + value.nanos_of_day); + } + const __int128 days = static_cast(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + const __int128 timestamp_micros = + days * MICROS_IN_DAY + value.nanos_of_day / NANOS_PER_MICROSECOND; + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet INT96 timestamp is outside the Doris 0001-9999 range"); + } + *result = static_cast(timestamp_micros); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/core/value/bitmap_value.h b/be/src/core/value/bitmap_value.h index 3b7b05d04a10c3..f18a7eb8fd3790 100644 --- a/be/src/core/value/bitmap_value.h +++ b/be/src/core/value/bitmap_value.h @@ -21,9 +21,13 @@ #include #include +#include +#include #include #include #include +#include +#include #include #include #include @@ -34,7 +38,9 @@ #include #include #include +#include #include +#include #include "common/config.h" #include "common/exception.h" @@ -99,12 +105,14 @@ namespace detail { class Roaring64MapSetBitForwardIterator; -// Forked from https://github.com/RoaringBitmap/CRoaring/blob/v0.2.60/cpp/roaring64map.hh +// Forked from https://github.com/RoaringBitmap/CRoaring/blob/v4.7.2/cpp/roaring/roaring64map.hh // What we change includes // - a custom serialization format is used inside read()/write()/getSizeInBytes() // - added clear() and is32BitsEnough() class Roaring64Map { public: + using RoaringMap = std::map; + /** * Create an empty bitmap */ @@ -141,41 +149,46 @@ class Roaring64Map { * Add value x * */ - void add(uint32_t x) { - roarings[0].add(x); - roarings[0].setCopyOnWrite(copyOnWrite); - } - void add(uint64_t x) { - roarings[highBytes(x)].add(lowBytes(x)); - roarings[highBytes(x)].setCopyOnWrite(copyOnWrite); - } + void add(uint32_t x) { lookupOrCreateInner(0).add(x); } - template + void add(uint64_t x) { lookupOrCreateInner(highBytes(x)).add(lowBytes(x)); } + + template && + !std::is_same_v, uint64_t>>> void addMany(size_t n_args, const T* vals) { if constexpr (sizeof(T) == sizeof(uint32_t)) { - auto& roaring = roarings[0]; - roaring.addMany(n_args, reinterpret_cast(vals)); - roaring.setCopyOnWrite(copyOnWrite); + lookupOrCreateInner(0).addMany(n_args, reinterpret_cast(vals)); } else if constexpr (sizeof(T) < sizeof(uint32_t)) { - auto& roaring = roarings[0]; std::vector values(n_args); for (size_t i = 0; i != n_args; ++i) { - values[i] = uint32_t(vals[i]); + values[i] = static_cast(vals[i]); } - roaring.addMany(n_args, values.data()); - roaring.setCopyOnWrite(copyOnWrite); + lookupOrCreateInner(0).addMany(n_args, values.data()); } else { - for (size_t lcv = 0; lcv < n_args; lcv++) { - roarings[highBytes(vals[lcv])].add(lowBytes(vals[lcv])); - roarings[highBytes(vals[lcv])].setCopyOnWrite(copyOnWrite); - } + addMany64(n_args, vals); } } - void addMany(size_t n_args, const uint64_t* vals) { + void addMany(size_t n_args, const uint64_t* vals) { addMany64(n_args, vals); } + + template + void addMany64(size_t n_args, const T* vals) { + // Potentially reduce outer map lookups by optimistically + // assuming that adjacent values will belong to the same inner bitmap. + roaring::Roaring* last_inner_bitmap = nullptr; + uint32_t last_value_high = 0; + roaring::BulkContext last_bulk_context; for (size_t lcv = 0; lcv < n_args; lcv++) { - roarings[highBytes(vals[lcv])].add(lowBytes(vals[lcv])); - roarings[highBytes(vals[lcv])].setCopyOnWrite(copyOnWrite); + auto value = static_cast(vals[lcv]); + auto value_high = highBytes(value); + auto value_low = lowBytes(value); + if (last_inner_bitmap == nullptr || value_high != last_value_high) { + last_inner_bitmap = &lookupOrCreateInner(value_high); + last_value_high = value_high; + last_bulk_context = roaring::BulkContext {}; + } + last_inner_bitmap->addBulk(last_bulk_context, value_low); } } @@ -192,7 +205,6 @@ class Roaring64Map { } /** * Return the largest value (if not empty) - * */ uint64_t maximum() const { for (auto roaring_iter = roarings.crbegin(); roaring_iter != roarings.crend(); @@ -201,14 +213,13 @@ class Roaring64Map { return uniteBytes(roaring_iter->first, roaring_iter->second.maximum()); } } - // we put std::numeric_limits<>::max/min in parenthesis + // we put std::numeric_limits<>::max/min in parentheses // to avoid a clash with the Windows.h header under Windows return (std::numeric_limits::min)(); } /** * Return the smallest value (if not empty) - * */ uint64_t minimum() const { for (auto roaring_iter = roarings.cbegin(); roaring_iter != roarings.cend(); @@ -217,7 +228,7 @@ class Roaring64Map { return uniteBytes(roaring_iter->first, roaring_iter->second.minimum()); } } - // we put std::numeric_limits<>::max/min in parenthesis + // we put std::numeric_limits<>::max/min in parentheses // to avoid a clash with the Windows.h header under Windows return (std::numeric_limits::max)(); } @@ -226,25 +237,75 @@ class Roaring64Map { * Check if value x is present */ bool contains(uint32_t x) const { - return roarings.count(0) == 0 ? false : roarings.at(0).contains(x); + auto iter = roarings.find(0); + if (iter == roarings.end()) { + return false; + } + return iter->second.contains(x); } bool contains(uint64_t x) const { - return roarings.count(highBytes(x)) == 0 ? false - : roarings.at(highBytes(x)).contains(lowBytes(x)); + auto iter = roarings.find(highBytes(x)); + if (iter == roarings.end()) { + return false; + } + return iter->second.contains(lowBytes(x)); } /** - * Compute the intersection between the current bitmap and the provided - * bitmap, + * Compute the intersection of the current bitmap and the provided bitmap, * writing the result in the current bitmap. The provided bitmap is not * modified. + * + * Performance hint: if you are computing the intersection between several + * bitmaps, two-by-two, it is best to start with the smallest bitmap. */ - Roaring64Map& operator&=(const Roaring64Map& r) { - for (auto& map_entry : roarings) { - if (r.roarings.count(map_entry.first) == 1) { - map_entry.second &= r.roarings.at(map_entry.first); - } else { - map_entry.second = roaring::Roaring(); + Roaring64Map& operator&=(const Roaring64Map& other) { + if (this == &other) { + // ANDing *this with itself is a no-op. + return *this; + } + + // Logic table summarizing what to do when a given outer key is + // present vs. absent from self and other. + // + // self other (self & other) work to do + // -------------------------------------------- + // absent absent empty None + // absent present empty None + // present absent empty Erase self + // present present empty or not Intersect self with other, but + // erase self if result is empty. + // + // Because there is only work to do when a key is present in 'self', the + // main for loop iterates over entries in 'self'. + + decltype(roarings.begin()) self_next; + for (auto self_iter = roarings.begin(); self_iter != roarings.end(); + self_iter = self_next) { + // Do the 'next' operation now, so we don't have to worry about + // invalidation of self_iter down below with the 'erase' operation. + self_next = std::next(self_iter); + + auto self_key = self_iter->first; + auto& self_bitmap = self_iter->second; + + auto other_iter = other.roarings.find(self_key); + if (other_iter == other.roarings.end()) { + // 'other' doesn't have self_key. In the logic table above, + // this reflects the case (self.present & other.absent). + // So, erase self. + roarings.erase(self_iter); + continue; + } + + // Both sides have self_key. In the logic table above, this reflects + // the case (self.present & other.present). So, intersect self with + // other. + const auto& other_bitmap = other_iter->second; + self_bitmap &= other_bitmap; + if (self_bitmap.isEmpty()) { + // ...but if intersection is empty, remove it altogether. + roarings.erase(self_iter); } } return *this; @@ -252,51 +313,178 @@ class Roaring64Map { /** * Compute the difference between the current bitmap and the provided - * bitmap, - * writing the result in the current bitmap. The provided bitmap is not - * modified. + * bitmap, writing the result in the current bitmap. The provided bitmap + * is not modified. */ - Roaring64Map& operator-=(const Roaring64Map& r) { - for (auto& map_entry : roarings) { - if (r.roarings.count(map_entry.first) == 1) { - map_entry.second -= r.roarings.at(map_entry.first); + Roaring64Map& operator-=(const Roaring64Map& other) { + if (this == &other) { + // Subtracting *this from itself results in the empty map. + roarings.clear(); + return *this; + } + + // Logic table summarizing what to do when a given outer key is + // present vs. absent from self and other. + // + // self other (self - other) work to do + // -------------------------------------------- + // absent absent empty None + // absent present empty None + // present absent unchanged None + // present present empty or not Subtract other from self, but + // erase self if result is empty + // + // Because there is only work to do when a key is present in both 'self' + // and 'other', the main while loop ping-pongs back and forth until it + // finds the next key that is the same on both sides. + + auto self_iter = roarings.begin(); + auto other_iter = other.roarings.cbegin(); + + while (self_iter != roarings.end() && other_iter != other.roarings.cend()) { + auto self_key = self_iter->first; + auto other_key = other_iter->first; + if (self_key < other_key) { + // Because self_key is < other_key, advance self_iter to the + // first point where self_key >= other_key (or end). + self_iter = roarings.lower_bound(other_key); + continue; + } + + if (self_key > other_key) { + // Because self_key is > other_key, advance other_iter to the + // first point where other_key >= self_key (or end). + other_iter = other.roarings.lower_bound(self_key); + continue; + } + + // Both sides have self_key. In the logic table above, this reflects + // the case (self.present & other.present). So subtract other from + // self. + auto& self_bitmap = self_iter->second; + const auto& other_bitmap = other_iter->second; + self_bitmap -= other_bitmap; + + if (self_bitmap.isEmpty()) { + // ...but if subtraction is empty, remove it altogether. + self_iter = roarings.erase(self_iter); + } else { + ++self_iter; } + ++other_iter; } return *this; } /** - * Compute the union between the current bitmap and the provided bitmap, + * Compute the union of the current bitmap and the provided bitmap, * writing the result in the current bitmap. The provided bitmap is not * modified. * * See also the fastunion function to aggregate many bitmaps more quickly. */ - Roaring64Map& operator|=(const Roaring64Map& r) { - for (const auto& map_entry : r.roarings) { - if (roarings.count(map_entry.first) == 0) { - roarings[map_entry.first] = map_entry.second; - roarings[map_entry.first].setCopyOnWrite(copyOnWrite); - } else { - roarings[map_entry.first] |= map_entry.second; + Roaring64Map& operator|=(const Roaring64Map& other) { + if (this == &other) { + // ORing *this with itself is a no-op. + return *this; + } + + // Logic table summarizing what to do when a given outer key is + // present vs. absent from self and other. + // + // self other (self | other) work to do + // -------------------------------------------- + // absent absent empty None + // absent present not empty Copy other to self and set flags + // present absent unchanged None + // present present not empty self |= other + // + // Because there is only work to do when a key is present in 'other', + // the main for loop iterates over entries in 'other'. + + for (const auto& other_entry : other.roarings) { + const auto& other_bitmap = other_entry.second; + + // Try to insert other_bitmap into self at other_key. We take + // advantage of the fact that std::map::insert will not overwrite an + // existing entry. + auto insert_result = roarings.insert(other_entry); + auto self_iter = insert_result.first; + auto insert_happened = insert_result.second; + auto& self_bitmap = self_iter->second; + + if (insert_happened) { + // Key was not present in self, so insert was performed above. + // In the logic table above, this reflects the case + // (self.absent | other.present). Because the copy has already + // happened, thanks to the 'insert' operation above, we just + // need to set the copyOnWrite flag. + self_bitmap.setCopyOnWrite(copyOnWrite); + continue; } + + // Both sides have self_key, and the insert was not performed. In + // the logic table above, this reflects the case + // (self.present & other.present). So OR other into self. + self_bitmap |= other_bitmap; } return *this; } /** - * Compute the symmetric union between the current bitmap and the provided - * bitmap, - * writing the result in the current bitmap. The provided bitmap is not - * modified. + * Compute the XOR of the current bitmap and the provided bitmap, writing + * the result in the current bitmap. The provided bitmap is not modified. */ - Roaring64Map& operator^=(const Roaring64Map& r) { - for (const auto& map_entry : r.roarings) { - if (roarings.count(map_entry.first) == 0) { - roarings[map_entry.first] = map_entry.second; - roarings[map_entry.first].setCopyOnWrite(copyOnWrite); - } else { - roarings[map_entry.first] ^= map_entry.second; + Roaring64Map& operator^=(const Roaring64Map& other) { + if (this == &other) { + // XORing *this with itself results in the empty map. + roarings.clear(); + return *this; + } + + // Logic table summarizing what to do when a given outer key is + // present vs. absent from self and other. + // + // self other (self ^ other) work to do + // -------------------------------------------- + // absent absent empty None + // absent present non-empty Copy other to self and set flags + // present absent unchanged None + // present present empty or not XOR other into self, but erase self + // if result is empty. + // + // Because there is only work to do when a key is present in 'other', + // the main for loop iterates over entries in 'other'. + + for (const auto& other_entry : other.roarings) { + const auto& other_bitmap = other_entry.second; + + // Try to insert other_bitmap into self at other_key. We take + // advantage of the fact that std::map::insert will not overwrite an + // existing entry. + auto insert_result = roarings.insert(other_entry); + auto self_iter = insert_result.first; + auto insert_happened = insert_result.second; + auto& self_bitmap = self_iter->second; + + if (insert_happened) { + // Key was not present in self, so insert was performed above. + // In the logic table above, this reflects the case + // (self.absent ^ other.present). Because the copy has already + // happened, thanks to the 'insert' operation above, we just + // need to set the copyOnWrite flag. + self_bitmap.setCopyOnWrite(copyOnWrite); + continue; + } + + // Both sides have self_key, and the insert was not performed. In + // the logic table above, this reflects the case + // (self.present ^ other.present). So XOR other into self. + self_bitmap ^= other_bitmap; + + if (self_bitmap.isEmpty()) { + // ...but if intersection is empty, remove it altogether. + roarings.erase(self_iter); } } return *this; @@ -339,6 +527,18 @@ class Roaring64Map { return card; } + bool intersect(const Roaring64Map& r) const { + const auto& smaller = roarings.size() <= r.roarings.size() ? roarings : r.roarings; + const auto& larger = roarings.size() <= r.roarings.size() ? r.roarings : roarings; + for (const auto& map_entry : smaller) { + auto it = larger.find(map_entry.first); + if (it != larger.cend() && map_entry.second.intersect(it->second)) { + return true; + } + } + return false; + } + /** * Computes the size of the union between two bitmaps. * @@ -394,6 +594,20 @@ class Roaring64Map { return card; } + bool isSubset(const Roaring64Map& r) const { + for (const auto& map_entry : roarings) { + if (map_entry.second.isEmpty()) { + continue; + } + auto roaring_iter = r.roarings.find(map_entry.first); + if (roaring_iter == r.roarings.cend()) + return false; + else if (!map_entry.second.isSubset(roaring_iter->second)) + return false; + } + return true; + } + /** * Returns true if the bitmap is empty (cardinality is zero). */ @@ -425,6 +639,22 @@ class Roaring64Map { : false; } + /** + * Convert the bitmap to a sorted array. Write the output to "ans", the + * caller is responsible to ensure that there is enough memory allocated + * (e.g., ans = new uint64_t[mybitmap.cardinality()];) + */ + void toUint64Array(uint64_t* ans) const { + // Annoyingly, VS 2017 marks std::accumulate() as [[nodiscard]] + (void)std::accumulate(roarings.cbegin(), roarings.cend(), ans, + [](uint64_t* previous, + const std::pair& map_entry) { + for (uint32_t low_bits : map_entry.second) + *previous++ = uniteBytes(map_entry.first, low_bits); + return previous; + }); + } + /** * Return true if the two bitmaps contain the same elements. */ @@ -436,10 +666,8 @@ class Roaring64Map { auto rhs_iter = r.roarings.cbegin(); auto rhs_cend = r.roarings.cend(); while (lhs_iter != lhs_cend && rhs_iter != rhs_cend) { - auto lhs_key = lhs_iter->first; - auto rhs_key = rhs_iter->first; - const auto& lhs_map = lhs_iter->second; - const auto& rhs_map = rhs_iter->second; + auto lhs_key = lhs_iter->first, rhs_key = rhs_iter->first; + const auto &lhs_map = lhs_iter->second, &rhs_map = rhs_iter->second; if (lhs_map.isEmpty()) { ++lhs_iter; continue; @@ -487,9 +715,9 @@ class Roaring64Map { } /** - * If needed, reallocate memory to shrink the memory usage. Returns - * the number of bytes saved. - */ + * If needed, reallocate memory to shrink the memory usage. + * Returns the number of bytes saved. + */ size_t shrinkToFit() { size_t savedBytes = 0; auto iter = roarings.begin(); @@ -497,7 +725,7 @@ class Roaring64Map { if (iter->second.isEmpty()) { // empty Roarings are 84 bytes savedBytes += 88; - iter = roarings.erase(iter); + roarings.erase(iter++); } else { savedBytes += iter->second.shrinkToFit(); iter++; @@ -507,13 +735,15 @@ class Roaring64Map { } /** - * Iterate over the bitmap elements. The function iterator is called once - * for all the values with ptr (can be nullptr) as the second parameter of each - * call. + * Iterate over the bitmap elements in order(start from the smallest one) + * and call iterator once for every element until the iterator function + * returns false. To iterate over all values, the iterator function should + * always return true. * - * roaring_iterator is simply a pointer to a function that returns bool - * (true means that the iteration should continue while false means that it - * should stop), and takes (uint32_t,void*) as inputs. + * The roaring_iterator64 parameter is a pointer to a function that + * returns bool (true means that the iteration should continue while false + * means that it should stop), and takes (uint64_t element, void* ptr) as + * inputs. */ void iterate(roaring::api::roaring_iterator64 iterator, void* ptr) const { for (const auto& map_entry : roarings) { @@ -665,8 +895,8 @@ class Roaring64Map { */ static Roaring64Map fastunion(size_t n, const Roaring64Map** inputs) { struct pq_entry { - phmap::btree_map::const_iterator iterator; - phmap::btree_map::const_iterator end; + RoaringMap::const_iterator iterator; + RoaringMap::const_iterator end; }; struct pq_comp { @@ -746,7 +976,7 @@ class Roaring64Map { const_iterator end() const; private: - phmap::btree_map roarings {}; + RoaringMap roarings {}; bool copyOnWrite {false}; static uint32_t highBytes(const uint64_t in) { return uint32_t(in >> 32); } static uint32_t lowBytes(const uint64_t in) { return uint32_t(in); } @@ -758,23 +988,60 @@ class Roaring64Map { } void emplaceOrInsert(const uint32_t key, roaring::Roaring&& value) { + // CRoaring's C++ move constructor bit-copies the underlying C struct and + // is unsafe with phmap::btree_map value relocation after operations such as shrinkToFit(). roarings.emplace(key, value); } + + roaring::Roaring& lookupOrCreateInner(uint32_t key) { + auto& bitmap = roarings[key]; + bitmap.setCopyOnWrite(copyOnWrite); + return bitmap; + } }; -// Forked from https://github.com/RoaringBitmap/CRoaring/blob/v0.4.0/cpp/roaring64map.hh +// Forked from https://github.com/RoaringBitmap/CRoaring/blob/v4.7.2/cpp/roaring/roaring64map.hh // Used to go through the set bits. Not optimally fast, but convenient. class Roaring64MapSetBitForwardIterator { public: + using iterator_category = std::forward_iterator_tag; + using pointer = uint64_t*; + using reference_type = uint64_t&; + using value_type = uint64_t; + using difference_type = int64_t; using type_of_iterator = Roaring64MapSetBitForwardIterator; /** * Provides the location of the set bit. */ - uint64_t operator*() const { + value_type operator*() const { return Roaring64Map::uniteBytes(map_iter->first, i.current_value); } + bool operator<(const type_of_iterator& o) const { + if (map_iter == map_end) return false; + if (o.map_iter == o.map_end) return true; + return **this < *o; + } + + bool operator<=(const type_of_iterator& o) const { + if (o.map_iter == o.map_end) return true; + if (map_iter == map_end) return false; + return **this <= *o; + } + + bool operator>(const type_of_iterator& o) const { + if (o.map_iter == o.map_end) return false; + if (map_iter == map_end) return true; + return **this > *o; + } + + bool operator>=(const type_of_iterator& o) const { + if (map_iter == map_end) return true; + if (o.map_iter == o.map_end) return false; + return **this >= *o; + } + type_of_iterator& operator++() { // ++i, must returned inc. value if (i.has_value) { roaring_advance_uint32_iterator(&i); @@ -798,22 +1065,6 @@ class Roaring64MapSetBitForwardIterator { return orig; } - bool move(const uint64_t& x) { - map_iter = p.lower_bound(Roaring64Map::highBytes(x)); - if (map_iter != p.cend()) { - roaring_init_iterator(&map_iter->second.roaring, &i); - if (map_iter->first == Roaring64Map::highBytes(x)) { - if (roaring_move_uint32_iterator_equalorlarger(&i, Roaring64Map::lowBytes(x))) - return true; - map_iter++; - if (map_iter == map_end) return false; - roaring_init_iterator(&map_iter->second.roaring, &i); - } - return true; - } - return false; - } - bool operator==(const Roaring64MapSetBitForwardIterator& o) const { if (map_iter == map_end && o.map_iter == o.map_end) return true; if (o.map_iter == o.map_end) return false; @@ -832,7 +1083,8 @@ class Roaring64MapSetBitForwardIterator { return *this; } - Roaring64MapSetBitForwardIterator(const Roaring64MapSetBitForwardIterator& r) = default; + Roaring64MapSetBitForwardIterator(const Roaring64MapSetBitForwardIterator& r) + : p(r.p), map_iter(r.map_iter), map_end(r.map_end), i(r.i) {} Roaring64MapSetBitForwardIterator(const Roaring64Map& parent, bool exhausted = false) : p(parent.roarings), map_end(parent.roarings.cend()) { @@ -850,9 +1102,9 @@ class Roaring64MapSetBitForwardIterator { } protected: - const phmap::btree_map& p; - phmap::btree_map::const_iterator map_iter {}; - phmap::btree_map::const_iterator map_end {}; + const Roaring64Map::RoaringMap& p; + Roaring64Map::RoaringMap::const_iterator map_iter {}; + Roaring64Map::RoaringMap::const_iterator map_end {}; roaring::api::roaring_uint32_iterator_t i {}; }; @@ -866,15 +1118,193 @@ inline Roaring64MapSetBitForwardIterator Roaring64Map::end() const { } // namespace detail +class BitmapValue; + +class BitmapSmallSet { +public: + using value_type = uint64_t; + using iterator = value_type*; + using const_iterator = const value_type*; + + static constexpr size_t INLINE_CAPACITY = 32; + static_assert(INLINE_CAPACITY <= std::numeric_limits::max(), + "INLINE_CAPACITY must fit into uint8_t"); + + BitmapSmallSet() = default; + BitmapSmallSet(const BitmapSmallSet&) = default; + BitmapSmallSet& operator=(const BitmapSmallSet&) = default; + BitmapSmallSet(BitmapSmallSet&& other) noexcept { *this = std::move(other); } + + BitmapSmallSet& operator=(BitmapSmallSet&& other) noexcept { + if (this == &other) { + return *this; + } + _values = other._values; + _size = other._size; + other.clear(); + return *this; + } + + size_t size() const { return _size; } + bool empty() const { return _size == 0; } + + iterator begin() { return _values.data(); } + iterator end() { return _values.data() + _size; } + const_iterator begin() const { return _values.data(); } + const_iterator end() const { return _values.data() + _size; } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + const value_type* data() const { return _values.data(); } + + void clear() { _size = 0; } + + bool contains(value_type value) const { return find_index(value) != npos; } + + // Use `insert/insert_many` only when the caller has proved the inline storage will not overflow. + // Paths that may increase cardinality should call `BitmapValue::add/add_many` instead. + void insert(value_type value) { + size_t pos = find_index(value); + if (pos != npos) { + return; + } + + DORIS_CHECK(_size < INLINE_CAPACITY) << "BitmapSmallSet overflow"; + + _values[_size] = value; + ++_size; + } + + template >> + void insert_many(const T* values, size_t n) { + DORIS_CHECK(_size + n <= INLINE_CAPACITY) << "BitmapSmallSet overflow"; + if (n == 0) { + return; + } + + if constexpr (std::is_same_v, value_type>) { + memcpy(_values.data() + _size, values, sizeof(value_type) * n); + } else { + std::array converted_values {}; + for (size_t i = 0; i < n; ++i) { + converted_values[i] = static_cast(values[i]); + } + memcpy(_values.data() + _size, converted_values.data(), sizeof(value_type) * n); + } + _size = static_cast(_size + n); + compact_unique(); + } + + template >> + bool try_insert_many(const T* values, size_t n) { + if (_size + n <= INLINE_CAPACITY) { + insert_many(values, n); + return true; + } + + BitmapSmallSet candidate = *this; + for (size_t i = 0; i < n; ++i) { + auto value = static_cast(values[i]); + if (candidate.contains(value)) { + continue; + } + if (candidate.size() >= INLINE_CAPACITY) { + return false; + } + candidate.insert(value); + } + *this = std::move(candidate); + return true; + } + + // Read serialized SET payload. Values are stored as fixed little-endian uint64_t. + bool read(const void* src, size_t n, bool deduplicate = false) { + DORIS_CHECK(n <= INLINE_CAPACITY) << "BitmapSmallSet overflow"; + clear(); + decode_fixed64_le_array(_values.data(), src, n); + _size = static_cast(n); + if (deduplicate) { + compact_unique(); + return true; + } + return has_unique_values(); + } + + void copy_to_sorted(value_type* dst) const { + if (_size > 0) { + memcpy(dst, _values.data(), sizeof(value_type) * _size); + } + std::sort(dst, dst + _size); + } + + size_t erase(value_type value) { + size_t pos = find_index(value); + if (pos == npos) { + return 0; + } + erase_at(pos); + return 1; + } + + value_type operator[](size_t index) const { return _values[index]; } + value_type& operator[](size_t index) { return _values[index]; } + +private: + friend class BitmapValue; + + size_t find_index(value_type value) const { + for (size_t i = 0; i < _size; ++i) { + if (_values[i] == value) { + return i; + } + } + return npos; + } + + void erase_at(size_t pos) { + DCHECK_LT(pos, _size); + _values[pos] = _values[_size - 1]; + --_size; + } + + bool has_unique_values() const { + for (size_t i = 0; i < _size; ++i) { + for (size_t j = i + 1; j < _size; ++j) { + if (_values[i] == _values[j]) { + return false; + } + } + } + return true; + } + + void compact_unique() { + size_t unique_size = 0; + for (size_t i = 0; i < _size; ++i) { + bool found = false; + for (size_t j = 0; j < unique_size; ++j) { + if (_values[j] == _values[i]) { + found = true; + break; + } + } + if (!found) { + _values[unique_size++] = _values[i]; + } + } + _size = static_cast(unique_size); + } + + std::array _values {}; + uint8_t _size = 0; + static constexpr size_t npos = static_cast(-1); +}; + // Represent the in-memory and on-disk structure of Doris's BITMAP data type. // Optimize for the case where the bitmap contains 0 or 1 element which is common // for streaming load scenario. class BitmapValueIterator; class BitmapValue { public: - template - using SetContainer = phmap::flat_hash_set; - // Construct an empty bitmap. BitmapValue() : _sv(0), _bitmap(nullptr), _type(EMPTY), _is_shared(false) { _set.clear(); } @@ -989,32 +1419,21 @@ class BitmapValue { break; } _is_shared = other._is_shared; + other._type = EMPTY; + other._is_shared = false; return *this; } // Construct a bitmap from given elements. - explicit BitmapValue(const std::vector& bits) : _is_shared(false) { - if (bits.size() == 0) { - _type = EMPTY; - return; - } - - if (bits.size() == 1) { - _type = SINGLE; - _sv = bits[0]; - return; - } + template >> + explicit BitmapValue(const std::vector& bits) : _is_shared(false) { + _type = EMPTY; + this->add_many(bits.data(), bits.size()); + } - if (!config::enable_set_in_bitmap_value || bits.size() > SET_TYPE_THRESHOLD) { - _type = BITMAP; - _prepare_bitmap_for_write(); - _bitmap->addMany(bits.size(), &bits[0]); - } else { - _type = SET; - for (auto v : bits) { - _set.insert(v); - } - } + explicit BitmapValue(const std::vector& bits) : _is_shared(false) { + _type = EMPTY; + add_many(bits.data(), bits.size()); } BitmapTypeCode::type get_type_code() const { @@ -1040,18 +1459,27 @@ class BitmapValue { __builtin_unreachable(); } - template + template >> void add_many(const T* values, const size_t count) { + if (count == 0) { + return; + } + switch (_type) { case EMPTY: if (count == 1) { - _sv = values[0]; + _sv = static_cast(values[0]); _type = SINGLE; - } else if (config::enable_set_in_bitmap_value && count < SET_TYPE_THRESHOLD) { - for (size_t i = 0; i != count; ++i) { - _set.insert(values[i]); + } else if (config::enable_set_in_bitmap_value) { + BitmapSmallSet new_set; + if (new_set.try_insert_many(values, count)) { + _set = std::move(new_set); + _type = SET; + } else { + _prepare_bitmap_for_write(); + _bitmap->addMany(count, values); + _type = BITMAP; } - _type = SET; } else { _prepare_bitmap_for_write(); _bitmap->addMany(count, values); @@ -1059,13 +1487,18 @@ class BitmapValue { } break; case SINGLE: - if (config::enable_set_in_bitmap_value && count < SET_TYPE_THRESHOLD) { - _set.insert(_sv); - for (size_t i = 0; i != count; ++i) { - _set.insert(values[i]); + if (config::enable_set_in_bitmap_value) { + BitmapSmallSet new_set; + new_set.insert(_sv); + if (new_set.try_insert_many(values, count)) { + _set = std::move(new_set); + _type = SET; + } else { + _prepare_bitmap_for_write(); + _bitmap->add(_sv); + _bitmap->addMany(count, values); + _type = BITMAP; } - _type = SET; - _convert_to_bitmap_if_need(); } else { _prepare_bitmap_for_write(); _bitmap->add(_sv); @@ -1078,10 +1511,10 @@ class BitmapValue { _bitmap->addMany(count, values); break; case SET: - for (size_t i = 0; i != count; ++i) { - _set.insert(values[i]); + if (!_set.try_insert_many(values, count)) { + _convert_set_to_bitmap(); + _bitmap->addMany(count, values); } - _convert_to_bitmap_if_need(); break; } } @@ -1093,8 +1526,9 @@ class BitmapValue { _sv = value; _type = SINGLE; } else { - _set.insert(value); _type = SET; + _set.clear(); + _set.insert(value); } break; case SINGLE: @@ -1103,6 +1537,7 @@ class BitmapValue { break; } if (config::enable_set_in_bitmap_value) { + _set.clear(); _set.insert(_sv); _set.insert(value); _type = SET; @@ -1118,8 +1553,15 @@ class BitmapValue { _bitmap->add(value); break; case SET: - _set.insert(value); - _convert_to_bitmap_if_need(); + if (_set.contains(value)) { + break; + } + if (_set.size() < SET_TYPE_THRESHOLD) { + _set.insert(value); + } else { + _convert_set_to_bitmap(); + _bitmap->add(value); + } break; } } @@ -1169,11 +1611,11 @@ class BitmapValue { _convert_to_smaller_type(); break; case SET: { - for (auto it = _set.begin(); it != _set.end();) { - if (rhs.contains(*it)) { - it = _set.erase(it); + for (size_t i = 0; i < _set.size();) { + if (rhs.contains(_set[i])) { + _set.erase_at(i); } else { - ++it; + ++i; } } _convert_to_smaller_type(); @@ -1193,18 +1635,16 @@ class BitmapValue { case BITMAP: _prepare_bitmap_for_write(); for (auto v : rhs._set) { - if (_bitmap->contains(v)) { - _bitmap->remove(v); - } + _bitmap->remove(v); } _convert_to_smaller_type(); break; case SET: { - for (auto it = _set.begin(); it != _set.end();) { - if (rhs.contains(*it)) { - it = _set.erase(it); + for (size_t i = 0; i < _set.size();) { + if (rhs.contains(_set[i])) { + _set.erase_at(i); } else { - ++it; + ++i; } } _convert_to_smaller_type(); @@ -1252,11 +1692,10 @@ class BitmapValue { case SET: { _prepare_bitmap_for_write(); *_bitmap = *rhs._bitmap; - - for (auto v : _set) { - _bitmap->add(v); - } + _bitmap->addMany(_set.size(), _set.data()); _type = BITMAP; + + _set.clear(); break; } } @@ -1268,34 +1707,19 @@ class BitmapValue { _type = SET; break; case SINGLE: { - if ((rhs._set.size() + rhs._set.contains(_sv) > SET_TYPE_THRESHOLD)) { - _prepare_bitmap_for_write(); - _bitmap->add(_sv); - for (auto v : rhs._set) { - _bitmap->add(v); - } - _type = BITMAP; - } else { - _set = rhs._set; - _set.insert(_sv); - _type = SET; - } + _set = rhs._set; + _type = SET; + this->add(_sv); break; } case BITMAP: _prepare_bitmap_for_write(); - for (auto v : rhs._set) { - _bitmap->add(v); - } + _bitmap->addMany(rhs._set.size(), rhs._set.data()); break; - case SET: { - for (auto v : rhs._set) { - _set.insert(v); - } - _convert_to_bitmap_if_need(); + case SET: + this->add_many(rhs._set.data(), rhs._set.size()); break; } - } break; } return *this; @@ -1304,7 +1728,7 @@ class BitmapValue { BitmapValue& fastunion(const std::vector& values) { std::vector bitmaps; std::vector single_values; - std::vector*> sets; + std::vector sets; for (const auto* value : values) { switch (value->_type) { case EMPTY: @@ -1338,9 +1762,7 @@ class BitmapValue { break; case SET: { *_bitmap = detail::Roaring64Map::fastunion(bitmaps.size(), bitmaps.data()); - for (auto v : _set) { - _bitmap->add(v); - } + _bitmap->addMany(_set.size(), _set.data()); _set.clear(); break; } @@ -1350,75 +1772,12 @@ class BitmapValue { if (!sets.empty()) { for (auto& set : sets) { - for (auto v : *set) { - _set.insert(v); - } - } - switch (_type) { - case EMPTY: - _type = SET; - break; - case SINGLE: { - _set.insert(_sv); - _type = SET; - break; - } - case BITMAP: - _prepare_bitmap_for_write(); - for (auto v : _set) { - _bitmap->add(v); - } - _type = BITMAP; - _set.clear(); - break; - case SET: { - break; - } - } - if (_type == SET) { - _convert_to_bitmap_if_need(); + this->add_many(set->data(), set->size()); } } - if (_type == EMPTY && single_values.size() == 1) { - if (config::enable_set_in_bitmap_value) { - _type = SET; - _set.insert(single_values[0]); - } else { - _sv = single_values[0]; - _type = SINGLE; - } - } else if (!single_values.empty()) { - switch (_type) { - case EMPTY: - case SINGLE: - if (config::enable_set_in_bitmap_value) { - _set.insert(single_values.cbegin(), single_values.cend()); - if (_type == SINGLE) { - _set.insert(_sv); - } - _type = SET; - _convert_to_bitmap_if_need(); - } else { - _prepare_bitmap_for_write(); - _bitmap->addMany(single_values.size(), single_values.data()); - if (_type == SINGLE) { - _bitmap->add(_sv); - } - _type = BITMAP; - _convert_to_smaller_type(); - } - break; - case BITMAP: { - _prepare_bitmap_for_write(); - _bitmap->addMany(single_values.size(), single_values.data()); - break; - } - case SET: - _set.insert(single_values.cbegin(), single_values.cend()); - _convert_to_bitmap_if_need(); - break; - } + if (!single_values.empty()) { + this->add_many(single_values.data(), single_values.size()); } return *this; @@ -1479,11 +1838,11 @@ class BitmapValue { _convert_to_smaller_type(); break; case SET: - for (auto it = _set.begin(); it != _set.end();) { - if (!rhs._bitmap->contains(*it)) { - it = _set.erase(it); + for (size_t i = 0; i < _set.size();) { + if (!rhs._bitmap->contains(_set[i])) { + _set.erase_at(i); } else { - ++it; + ++i; } } _convert_to_smaller_type(); @@ -1512,11 +1871,11 @@ class BitmapValue { _convert_to_smaller_type(); break; case SET: - for (auto it = _set.begin(); it != _set.end();) { - if (!rhs._set.contains(*it)) { - it = _set.erase(it); + for (size_t i = 0; i < _set.size();) { + if (!rhs._set.contains(_set[i])) { + _set.erase_at(i); } else { - ++it; + ++i; } } _convert_to_smaller_type(); @@ -1546,22 +1905,22 @@ class BitmapValue { if (_sv == rhs._sv) { _type = EMPTY; } else { - add(rhs._sv); + this->add(rhs._sv); } break; case BITMAP: if (!_bitmap->contains(rhs._sv)) { - add(rhs._sv); + this->add(rhs._sv); } else { - _prepare_bitmap_for_write(); - _bitmap->remove(rhs._sv); + this->remove(rhs._sv); } + _convert_to_smaller_type(); break; case SET: if (!_set.contains(rhs._sv)) { - _set.insert(rhs._sv); + this->add(rhs._sv); } else { - _set.erase(rhs._sv); + this->remove(rhs._sv); } break; } @@ -1583,7 +1942,7 @@ class BitmapValue { if (!rhs._bitmap->contains(_sv)) { _bitmap->add(_sv); } else { - _bitmap->remove(_sv); + this->remove(_sv); } break; case BITMAP: @@ -1601,6 +1960,7 @@ class BitmapValue { _bitmap->add(v); } } + _set.clear(); _type = BITMAP; _convert_to_smaller_type(); break; @@ -1614,12 +1974,12 @@ class BitmapValue { break; case SINGLE: _set = rhs._set; - if (!rhs._set.contains(_sv)) { - _set.insert(_sv); + _type = SET; + if (_set.contains(_sv)) { + this->remove(_sv); } else { - _set.erase(_sv); + this->add(_sv); } - _type = SET; break; case BITMAP: _prepare_bitmap_for_write(); @@ -1633,14 +1993,27 @@ class BitmapValue { _convert_to_smaller_type(); break; case SET: - for (auto v : rhs._set) { - if (_set.contains(v)) { - _set.erase(v); - } else { - _set.insert(v); + if (this == &rhs) { + reset(); + break; + } + + std::array values_to_add {}; + size_t add_count = 0; + for (size_t i = 0; i < rhs._set.size(); ++i) { + auto v = rhs._set[i]; + if (_set.erase(v) == 0) { + values_to_add[add_count++] = v; } } - _convert_to_smaller_type(); + + if (_set.size() + add_count <= SET_TYPE_THRESHOLD) { + _set.insert_many(values_to_add.data(), add_count); + _convert_to_smaller_type(); + } else { + _convert_set_to_bitmap(); + _bitmap->addMany(add_count, values_to_add.data()); + } break; } break; @@ -1663,8 +2036,112 @@ class BitmapValue { return false; } - // true if contains a value that belongs to the range [left, right]. - bool contains_any(uint64_t left, uint64_t right) const; + bool intersects(const BitmapValue& rhs) const { + switch (rhs._type) { + case EMPTY: + return false; + case SINGLE: + return contains(rhs._sv); + case BITMAP: + switch (_type) { + case EMPTY: + return false; + case SINGLE: + return rhs._bitmap->contains(_sv); + case BITMAP: + return _bitmap->intersect(*rhs._bitmap); + case SET: + for (auto v : _set) { + if (rhs._bitmap->contains(v)) { + return true; + } + } + return false; + } + break; + case SET: + switch (_type) { + case EMPTY: + return false; + case SINGLE: + return rhs._set.contains(_sv); + case BITMAP: + for (auto v : rhs._set) { + if (_bitmap->contains(v)) { + return true; + } + } + return false; + case SET: { + const auto& smaller = _set.size() <= rhs._set.size() ? _set : rhs._set; + const auto& larger = _set.size() <= rhs._set.size() ? rhs._set : _set; + for (auto v : smaller) { + if (larger.contains(v)) { + return true; + } + } + return false; + } + } + } + return false; + } + + bool contains_all(const BitmapValue& rhs) const { + if (rhs.cardinality() == 0) { + return true; + } + + switch (rhs._type) { + case EMPTY: + return true; + case SINGLE: + return contains(rhs._sv); + case BITMAP: + switch (_type) { + case EMPTY: + return false; + case SINGLE: + return rhs._bitmap->cardinality() == 0 || + (rhs._bitmap->cardinality() == 1 && rhs._bitmap->contains(_sv)); + case SET: + if (rhs._bitmap->cardinality() > _set.size()) { + return false; + } + for (auto v : *rhs._bitmap) { + if (!_set.contains(v)) { + return false; + } + } + return true; + case BITMAP: + return rhs._bitmap->isSubset(*_bitmap); + } + break; + case SET: + switch (_type) { + case EMPTY: + return false; + case SINGLE: + return rhs._set.size() == 1 && rhs._set.contains(_sv); + case BITMAP: + for (auto v : rhs._set) { + if (!_bitmap->contains(v)) { + return false; + } + } + return true; + case SET: + for (auto v : rhs._set) { + if (!_set.contains(v)) { + return false; + } + } + return true; + } + } + return false; + } uint64_t cardinality() const { switch (_type) { @@ -1732,8 +2209,10 @@ class BitmapValue { } case SET: { uint64_t cardinality = 0; - for (auto v : _set) { - if (rhs._set.contains(v)) { + const auto& smaller = _set.size() <= rhs._set.size() ? _set : rhs._set; + const auto& larger = _set.size() <= rhs._set.size() ? rhs._set : _set; + for (auto v : smaller) { + if (larger.contains(v)) { ++cardinality; } } @@ -1795,13 +2274,7 @@ class BitmapValue { return cardinality; } case SET: { - uint64_t cardinality = _set.size(); - for (auto v : _set) { - if (!rhs._set.contains(v)) { - ++cardinality; - } - } - return cardinality; + return _set.size() + rhs._set.size() - and_cardinality(rhs); } } } @@ -1859,13 +2332,7 @@ class BitmapValue { return cardinality; } case SET: { - uint64_t cardinality = _set.size(); - for (auto v : rhs._set) { - if (_set.contains(v)) { - cardinality -= 1; - } - } - return cardinality; + return _set.size() - and_cardinality(rhs); } } } @@ -1924,9 +2391,14 @@ class BitmapValue { ++dst; *dst = static_cast(_set.size()); ++dst; - for (auto v : _set) { - encode_fixed64_le(reinterpret_cast(dst), v); - dst += sizeof(uint64_t); + { + // SET serialization does not guarantee a canonical value order. The old phmap-backed + // SET order also depended on bucket state, so write the current inline order directly. + // TypeCode::SET(1 byte) | count(1 byte) | uint64 values in little-endian order. + for (auto v : _set) { + encode_fixed64_le(reinterpret_cast(dst), v); + dst += sizeof(uint64_t); + } } break; case BITMAP: @@ -1938,6 +2410,7 @@ class BitmapValue { // Deserialize a bitmap value from `src`. // Return false if `src` begins with unknown type code, true otherwise. bool deserialize(const char* src) { + reset(); switch (*src) { case BitmapTypeCode::EMPTY: _type = EMPTY; @@ -1947,6 +2420,7 @@ class BitmapValue { _sv = decode_fixed32_le(reinterpret_cast(src + 1)); if (config::enable_set_in_bitmap_value) { _type = SET; + _set.clear(); _set.insert(_sv); } break; @@ -1955,6 +2429,7 @@ class BitmapValue { _sv = decode_fixed64_le(reinterpret_cast(src + 1)); if (config::enable_set_in_bitmap_value) { _type = SET; + _set.clear(); _set.insert(_sv); } break; @@ -1980,51 +2455,34 @@ class BitmapValue { throw Exception(ErrorCode::INTERNAL_ERROR, "bitmap value with incorrect set count, count: {}", count); } - _set.reserve(count); - for (uint8_t i = 0; i != count; ++i, src += sizeof(uint64_t)) { - _set.insert(decode_fixed64_le(reinterpret_cast(src))); - } - if (_set.size() != count) { + if (!_set.read(src, count)) { throw Exception(ErrorCode::INTERNAL_ERROR, - "bitmap value with incorrect set count, count: {}, set size: {}", - count, _set.size()); + "bitmap value with duplicated set values, count: {}", count); } + src += sizeof(uint64_t) * count; if (!config::enable_set_in_bitmap_value) { - _prepare_bitmap_for_write(); - for (auto v : _set) { - _bitmap->add(v); - } - _type = BITMAP; - _set.clear(); + _convert_set_to_bitmap(); } break; } case BitmapTypeCode::SET_V2: { - uint32_t size = 0; - memcpy(&size, src + 1, sizeof(uint32_t)); + uint32_t size = decode_fixed32_le(reinterpret_cast(src + 1)); src += sizeof(uint32_t) + 1; if (!config::enable_set_in_bitmap_value || size > SET_TYPE_THRESHOLD) { _type = BITMAP; _prepare_bitmap_for_write(); - for (int i = 0; i < size; ++i) { - uint64_t key {}; - memcpy(&key, src, sizeof(uint64_t)); - _bitmap->add(key); - src += sizeof(uint64_t); + if (size > 0) { + std::vector values(size); + decode_fixed64_le_array(values.data(), src, size); + _bitmap->addMany(size, values.data()); + src += sizeof(uint64_t) * size; } } else { _type = SET; - _set.reserve(size); - - for (int i = 0; i < size; ++i) { - uint64_t key {}; - memcpy(&key, src, sizeof(uint64_t)); - _set.insert(key); - src += sizeof(uint64_t); - } + _set.read(src, size, true); } break; } @@ -2087,10 +2545,11 @@ class BitmapValue { } iter_ctx; iter_ctx.ss = &ss; - std::vector values(_set.begin(), _set.end()); - std::sort(values.begin(), values.end()); + std::array values {}; + _set.copy_to_sorted(values.data()); - for (auto v : values) { + for (size_t i = 0; i < _set.size(); ++i) { + auto v = values[i]; if (iter_ctx.first) { iter_ctx.first = false; } else { @@ -2171,14 +2630,12 @@ class BitmapValue { } case SET: { int64_t count = 0; - std::vector values(_set.begin(), _set.end()); - std::sort(values.begin(), values.end()); - for (auto it = values.begin(); it != values.end(); ++it) { - if (*it < range_start || *it >= range_end) { - continue; + for (size_t i = 0; i < _set.size(); ++i) { + uint64_t v = _set[i]; + if (v >= range_start && v < range_end) { + ret_bitmap->add(v); + ++count; } - ret_bitmap->add(*it); - ++count; } return count; } @@ -2194,11 +2651,14 @@ class BitmapValue { */ int64_t sub_limit(const int64_t& range_start, const int64_t& cardinality_limit, BitmapValue* ret_bitmap) const { + if (cardinality_limit <= 0) { + return 0; + } switch (_type) { case EMPTY: return 0; case SINGLE: { - //only single value, so range_start must less than _sv + // Only single value, so range_start must be less than or equal to _sv. if (range_start > _sv) { return 0; } else { @@ -2222,22 +2682,26 @@ class BitmapValue { return count; } case SET: { - int64_t count = 0; - - std::vector values(_set.begin(), _set.end()); - std::sort(values.begin(), values.end()); - for (auto it = values.begin(); it != values.end(); ++it) { - if (*it < range_start) { + std::array values {}; + size_t count = 0; + for (size_t i = 0; i < _set.size(); ++i) { + auto v = _set[i]; + if (v < range_start) { continue; } - if (count < cardinality_limit) { - ret_bitmap->add(*it); - ++count; - } else { - break; - } + values[count++] = v; } - return count; + + const auto output_count = std::min(count, static_cast(cardinality_limit)); + if (output_count == 0) { + return 0; + } + if (output_count < count) { + std::nth_element(values.begin(), values.begin() + output_count, + values.begin() + count); + } + ret_bitmap->add_many(values.data(), output_count); + return output_count; } } return 0; @@ -2250,6 +2714,10 @@ class BitmapValue { */ int64_t offset_limit(const int64_t& offset, const int64_t& limit, BitmapValue* ret_bitmap) const { + if (limit <= 0) { + return 0; + } + switch (_type) { case EMPTY: return 0; @@ -2266,7 +2734,7 @@ class BitmapValue { break; } if (_type == BITMAP) { - if (std::abs(offset) >= _bitmap->cardinality()) { + if (std::abs(offset) > _bitmap->cardinality()) { return 0; } int64_t abs_offset = offset; @@ -2285,7 +2753,7 @@ class BitmapValue { } return count; } else { - if (std::abs(offset) > _set.size()) { + if (std::abs(offset) > _set.size() || limit <= 0) { return 0; } @@ -2293,25 +2761,28 @@ class BitmapValue { if (offset < 0) { abs_offset = _set.size() + offset; } + if (abs_offset >= _set.size()) { + return 0; + } - std::vector values(_set.begin(), _set.end()); - std::sort(values.begin(), values.end()); + std::array values {}; + memcpy(values.data(), _set.data(), sizeof(uint64_t) * _set.size()); - int64_t count = 0; - size_t index = 0; - for (auto v : values) { - if (index < abs_offset) { - ++index; - continue; - } - if (count == limit || index == values.size()) { - break; - } - ++count; - ++index; - ret_bitmap->add(v); + const auto start = static_cast(abs_offset); + const auto end = std::min(_set.size(), start + static_cast(limit)); + if (end == start) { + return 0; } - return count; + + if (end < _set.size()) { + std::nth_element(values.begin(), values.begin() + end, + values.begin() + _set.size()); + } + if (start > 0) { + std::nth_element(values.begin(), values.begin() + start, values.begin() + end); + } + ret_bitmap->add_many(values.data() + start, end - start); + return end - start; } } @@ -2325,16 +2796,17 @@ class BitmapValue { break; } case BITMAP: { - for (auto it = _bitmap->begin(); it != _bitmap->end(); ++it) { - data.emplace_back(*it); - } + const auto old_size = data.size(); + const auto cardinality = _bitmap->cardinality(); + data.resize(old_size + cardinality); + _bitmap->toUint64Array(reinterpret_cast(data.data() + old_size)); break; } case SET: { - std::vector values(_set.begin(), _set.end()); - std::sort(values.begin(), values.end()); - for (auto v : values) { - data.emplace_back(v); + std::array values {}; + _set.copy_to_sorted(values.data()); + for (size_t i = 0; i < _set.size(); ++i) { + data.emplace_back(values[i]); } break; } @@ -2354,7 +2826,6 @@ class BitmapValue { b_iterator begin() const; b_iterator end() const; - b_iterator lower_bound(uint64_t val) const; private: void _convert_to_smaller_type() { @@ -2372,6 +2843,7 @@ class BitmapValue { _sv = _bitmap->minimum(); } else { _type = SET; + _set.clear(); for (auto v : *_bitmap) { _set.insert(v); } @@ -2382,6 +2854,8 @@ class BitmapValue { _type = SINGLE; _sv = *_set.begin(); _set.clear(); + } else if (_set.empty()) { + _type = EMPTY; } } } @@ -2426,14 +2900,12 @@ class BitmapValue { _is_shared = false; } - void _convert_to_bitmap_if_need() { - if (_type != SET || _set.size() <= SET_TYPE_THRESHOLD) { - return; - } + // Promote the current SET payload to roaring. Use this only when the caller has already + // decided SET is no longer the right representation, for example add() detects a new value + // cannot fit in the inline set, or an operation intentionally switches to roaring first. + void _convert_set_to_bitmap() { _prepare_bitmap_for_write(); - for (auto v : _set) { - _bitmap->add(v); - } + _bitmap->addMany(_set.size(), _set.data()); _type = BITMAP; _set.clear(); } @@ -2447,7 +2919,7 @@ class BitmapValue { uint64_t _sv = 0; // store the single value when _type == SINGLE // !FIXME: We should rethink the logic about _bitmap and _is_shared mutable std::shared_ptr _bitmap; // used when _type == BITMAP - SetContainer _set; + BitmapSmallSet _set; BitmapDataType _type {EMPTY}; // Indicate whether the state is shared among multi BitmapValue object mutable bool _is_shared = true; @@ -2573,34 +3045,10 @@ class BitmapValueIterator { bool operator!=(const BitmapValueIterator& other) const { return !(*this == other); } - /** - * Move the iterator to the first value >= `val`. - */ - BitmapValueIterator& move(uint64_t val) { - switch (_bitmap._type) { - case BitmapValue::BitmapDataType::SINGLE: - if (_sv < val) { - _end = true; - } - break; - case BitmapValue::BitmapDataType::BITMAP: - if (!_iter->move(val)) { - _end = true; - } - break; - case BitmapValue::BitmapDataType::SET: { - throw Exception(Status::FatalError("BitmapValue with set do not support move")); - } - default: - break; - } - return *this; - } - private: const BitmapValue& _bitmap; detail::Roaring64MapSetBitForwardIterator* _iter = nullptr; - BitmapValue::SetContainer::const_iterator _set_iter; + BitmapSmallSet::const_iterator _set_iter; uint64_t _sv = 0; bool _end = false; }; @@ -2613,25 +3061,4 @@ inline BitmapValueIterator BitmapValue::end() const { return {*this, true}; } -inline BitmapValueIterator BitmapValue::lower_bound(uint64_t val) const { - return BitmapValueIterator(*this).move(val); -} - -inline bool BitmapValue::contains_any(uint64_t left, uint64_t right) const { - if (left > right) { - return false; - } - - if (_type == SET) { - for (auto v : _set) { - if (v >= left && v <= right) { - return true; - } - } - return false; - } - auto it = lower_bound(left); - return it != end() && *it <= right; -} - } // namespace doris diff --git a/be/src/exec/common/variant_util.cpp b/be/src/exec/common/variant_util.cpp index 4ba50f5f57d3b3..86c274875da40b 100644 --- a/be/src/exec/common/variant_util.cpp +++ b/be/src/exec/common/variant_util.cpp @@ -72,6 +72,7 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type/get_least_supertype.h" #include "core/data_type/primitive_type.h" +#include "core/data_type/storage_field_type.h" #include "core/field.h" #include "core/typeid_cast.h" #include "core/types.h" @@ -1902,26 +1903,26 @@ static void append_field_to_binary_chars(const Field& field, ColumnString::Chars } case PrimitiveType::TYPE_BOOLEAN: { append_binary_type(chars, - TabletColumn::get_field_type_by_type(PrimitiveType::TYPE_BOOLEAN)); + primitive_type_to_storage_field_type(PrimitiveType::TYPE_BOOLEAN)); const auto v = static_cast(field.get()); append_binary_bytes(chars, &v, sizeof(UInt8)); return; } case PrimitiveType::TYPE_BIGINT: { - append_binary_type(chars, TabletColumn::get_field_type_by_type(PrimitiveType::TYPE_BIGINT)); + append_binary_type(chars, primitive_type_to_storage_field_type(PrimitiveType::TYPE_BIGINT)); const auto v = field.get(); append_binary_bytes(chars, &v, sizeof(Int64)); return; } case PrimitiveType::TYPE_LARGEINT: { append_binary_type(chars, - TabletColumn::get_field_type_by_type(PrimitiveType::TYPE_LARGEINT)); + primitive_type_to_storage_field_type(PrimitiveType::TYPE_LARGEINT)); const auto v = field.get(); append_binary_bytes(chars, &v, sizeof(int128_t)); return; } case PrimitiveType::TYPE_DOUBLE: { - append_binary_type(chars, TabletColumn::get_field_type_by_type(PrimitiveType::TYPE_DOUBLE)); + append_binary_type(chars, primitive_type_to_storage_field_type(PrimitiveType::TYPE_DOUBLE)); const auto v = field.get(); append_binary_bytes(chars, &v, sizeof(Float64)); return; diff --git a/be/src/exec/operator/cache_sink_operator.cpp b/be/src/exec/operator/cache_sink_operator.cpp index 97c42b99c252c7..b8c7204ec1a432 100644 --- a/be/src/exec/operator/cache_sink_operator.cpp +++ b/be/src/exec/operator/cache_sink_operator.cpp @@ -54,12 +54,13 @@ Status CacheSinkOperatorX::sink_impl(RuntimeState* state, Block* in_block, bool SCOPED_TIMER(local_state.exec_time_counter()); COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)in_block->rows()); - if (in_block->rows() > 0) { - RETURN_IF_ERROR(local_state._shared_state->data_queue.push_block( - Block::create_unique(std::move(*in_block)), 0)); - } - if (UNLIKELY(eos)) { - local_state._shared_state->data_queue.set_finish(0); + if (in_block->rows() > 0 || eos) { + std::unique_ptr output_block; + if (in_block->rows() > 0) { + output_block = Block::create_unique(std::move(*in_block)); + } + RETURN_IF_ERROR( + local_state._shared_state->data_queue.push_block(std::move(output_block), 0, eos)); } return Status::OK(); } diff --git a/be/src/exec/operator/cache_source_operator.cpp b/be/src/exec/operator/cache_source_operator.cpp index ec7d947680a6f8..d83c83788c0eb7 100644 --- a/be/src/exec/operator/cache_source_operator.cpp +++ b/be/src/exec/operator/cache_source_operator.cpp @@ -35,11 +35,11 @@ Status CacheSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { ((DataQueueSharedState*)_dependency->shared_state()) ->data_queue.set_source_dependency(_shared_state->source_deps.front()); const auto& scan_ranges = info.scan_ranges; - bool hit_cache = false; - const auto& cache_param = _parent->cast()._cache_param; + auto& parent = _parent->cast(); + const auto& cache_param = parent._cache_param; // 1. init the slot orders - const auto& tuple_descs = _parent->cast().row_desc().tuple_descriptors(); + const auto& tuple_descs = parent.row_desc().tuple_descriptors(); for (auto tuple_desc : tuple_descs) { for (auto slot_desc : tuple_desc->slots()) { if (cache_param.output_slot_mapping.find(slot_desc->id()) != @@ -53,8 +53,6 @@ Status CacheSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { } } - // 2. build cache key by digest_tablet_id - RETURN_IF_ERROR(QueryCache::build_cache_key(scan_ranges, cache_param, &_cache_key, &_version)); std::vector cache_tablet_ids; cache_tablet_ids.reserve(scan_ranges.size()); for (const auto& scan_range : scan_ranges) { @@ -70,14 +68,40 @@ Status CacheSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { } custom_profile()->add_info_string("CacheTabletId", tablet_ids_str); - // 3. lookup the cache and find proper slot order - if (!cache_param.force_refresh_query_cache) { - hit_cache = _global_cache->lookup(_cache_key, _version, &_query_cache_handle); + // 2. consume the per-instance cache decision. It is made exactly once for + // this instance and shared with the olap scan operator, so both operators + // always act consistently (e.g. the scan skips scanning if and only if this + // operator emits the cached blocks) -- whatever their init order is, and + // even if the entry gets evicted in between (the decision pins it). + if (parent._query_cache_runtime == nullptr) { + // The fragment context always creates the runtime together with this + // operator. Degrading to a pass-through here would silently drop data + // if the paired scan operator still made a HIT decision (it skips + // scanning while nothing emits the entry), so a broken setup must + // fail loudly, mirroring the scan side. + return Status::InternalError( + "query cache runtime is absent at the cache source, node_id={}", + cache_param.node_id); } - custom_profile()->add_info_string("HitCache", std::to_string(hit_cache)); - if (hit_cache) { - _hit_cache_results = _query_cache_handle.get_cache_result(); - auto hit_cache_slot_orders = _query_cache_handle.get_cache_slot_orders(); + _global_cache = parent._query_cache_runtime->cache(); + _cache_decision = parent._query_cache_runtime->get_or_make_decision(scan_ranges); + _cache_key = _cache_decision->cache_key; + _version = _cache_decision->current_version; + + using Mode = QueryCacheInstanceDecision::Mode; + const bool hit_cache = _cache_decision->mode == Mode::HIT; + _is_incremental = _cache_decision->mode == Mode::INCREMENTAL; + // HIT emits the entry unchanged, so there is nothing to write back; both + // MISS and INCREMENTAL rebuild the entry (from scratch / by merge), unless + // the decision already knows the merged entry could never fit the entry + // limits (write_back_feasible == false). + _need_insert_cache = + _cache_decision->key_valid && !hit_cache && _cache_decision->write_back_feasible; + _insert_delta_count = _is_incremental ? _cache_decision->cached_delta_count + 1 : 0; + + if (hit_cache || _is_incremental) { + _hit_cache_results = _cache_decision->handle.get_cache_result(); + auto hit_cache_slot_orders = _cache_decision->handle.get_cache_slot_orders(); if (_slot_orders != *hit_cache_slot_orders) { for (auto slot_id : _slot_orders) { @@ -96,6 +120,18 @@ Status CacheSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { } } + custom_profile()->add_info_string("HitCache", std::to_string(hit_cache)); + custom_profile()->add_info_string("HitCacheStale", std::to_string(_is_incremental)); + if (_is_incremental) { + custom_profile()->add_info_string( + "IncrementalDeltaVersions", + fmt::format("({}, {}]", _cache_decision->cached_version, _version)); + } + if (!_cache_decision->incremental_fallback_reason.empty()) { + custom_profile()->add_info_string("IncrementalFallbackReason", + _cache_decision->incremental_fallback_reason); + } + return Status::OK(); } @@ -107,13 +143,41 @@ Status CacheSourceLocalState::open(RuntimeState* state) { return Status::OK(); } +bool CacheSourceLocalState::_account_write_back(int64_t rows, int64_t bytes) { + _current_query_cache_rows += rows; + _current_query_cache_bytes += bytes; + const auto& cache_param = _parent->cast()._cache_param; + if (cache_param.entry_max_bytes < _current_query_cache_bytes || + cache_param.entry_max_rows < _current_query_cache_rows) { + // over the max bytes/rows: pass the data through, no need to do cache + _local_cache_blocks.clear(); + _need_insert_cache = false; + return false; + } + return true; +} + +Status CacheSourceLocalState::_append_block_for_write_back(const Block& block) { + if (!_account_write_back(block.rows(), block.allocated_bytes())) { + return Status::OK(); + } + // Zero-copy snapshot: inserting shares the COW column pointers instead + // of cloning the payload. The cached columns stay alive (and immutable) + // through the decision's entry pin until QueryCache::insert() deep-copies + // the accumulated set once. + auto& kept = _local_cache_blocks.emplace_back(Block::create_unique()); + for (const auto& column : block.get_columns_with_type_and_name()) { + kept->insert(column); + } + return Status::OK(); +} + std::string CacheSourceLocalState::debug_string(int indentation_level) const { fmt::memory_buffer debug_string_buffer; fmt::format_to(debug_string_buffer, "{}", Base::debug_string(indentation_level)); if (_shared_state) { - fmt::format_to(debug_string_buffer, ", data_queue: (is_all_finish = {}, has_data = {})", - _shared_state->data_queue.is_all_finish(), - _shared_state->data_queue.has_more_data()); + fmt::format_to(debug_string_buffer, ", data_queue: {}", + _shared_state->data_queue.debug_string()); } return fmt::to_string(debug_string_buffer); } @@ -125,8 +189,72 @@ Status CacheSourceOperatorX::get_block_impl(RuntimeState* state, Block* block, b block->clear_column_data(_row_descriptor.num_materialized_slots()); bool need_clone_empty = block->columns() == 0; - if (local_state._hit_cache_results == nullptr) { - Defer insert_cache([&] { + const bool has_cached_block = + local_state._hit_cache_results != nullptr && + local_state._hit_cache_pos < local_state._hit_cache_results->size(); + + if (has_cached_block) { + // Emit one cached block: the whole result on HIT, or the cached part + // ahead of the delta on INCREMENTAL. Both the cached blocks and the + // delta blocks are partial aggregation states, so the upstream merge + // aggregation combines them into the final result. + // Note: this operator is only scheduled once the data queue has data or + // finished, so on INCREMENTAL the cached blocks are not emitted before + // the first delta block arrives. Making the source dependency initially + // ready for HIT/INCREMENTAL would overlap emitting the cached part with + // the delta scan -- a possible future latency optimization. + const auto& hit_cache_block = + local_state._hit_cache_results->at(local_state._hit_cache_pos++); + // Reorder the cached block to this query's slot order BEFORE merging + // (a zero-copy view: inserting reuses the COW column pointers). + // Merging first and permuting afterwards is an order trap on the + // second cached block whenever the reused output block keeps its + // schema between pulls: a cached-order block then merges positionally + // into query-order columns -- a type error for heterogeneous slots, + // silently misplaced data for same-typed ones. Today the trap stays + // latent by accident: this operator is built without a plan node, so + // its own _row_descriptor reports zero materialized slots and the + // clear_column_data() above wipes the whole block every pull. The + // pipeline driver's clears are schema-keeping though, so giving this + // operator a real row descriptor (or dropping the redundant-looking + // wipe above) would arm it; reordering the source keeps the merge + // order-aligned under both block shapes. + Block reordered_cache_block; + const Block* cache_block_to_merge = hit_cache_block.get(); + if (!local_state._hit_cache_column_orders.empty()) { + for (auto loc : local_state._hit_cache_column_orders) { + reordered_cache_block.insert(hit_cache_block->get_by_position(loc)); + } + cache_block_to_merge = &reordered_cache_block; + } + if (need_clone_empty) { + *block = cache_block_to_merge->clone_empty(); + } + { + ScopedMutableBlock scoped_mutable_block(block); + auto& mutable_block = scoped_mutable_block.mutable_block(); + RETURN_IF_ERROR(mutable_block.merge(*cache_block_to_merge)); + scoped_mutable_block.restore(); + } + if (local_state._is_incremental && local_state._need_insert_cache) { + // Snapshot the cached block (already in this query's slot order) + // for the write-back, so the new entry holds "cached + delta" + // under one consistent slot order. The snapshot shares the COW + // columns of the pinned entry, so no payload is copied before + // the insert-time materialization. + RETURN_IF_ERROR(local_state._append_block_for_write_back(*cache_block_to_merge)); + } + } else if (local_state._hit_cache_results != nullptr && !local_state._is_incremental) { + // HIT: all cached blocks are emitted. + *eos = true; + } else { + // MISS, or the delta phase of INCREMENTAL after the cached blocks. + // The entry is committed only from the explicit success paths below, + // never during error unwinding: on the final delta block *eos is set + // BEFORE the block is merged, so a deferred/unconditional commit + // would publish an entry that is missing the failed block under the + // current version, and a later exact hit would silently serve it. + auto commit_entry_if_finished = [&]() { if (*eos) { local_state.custom_profile()->add_info_string( "InsertCache", std::to_string(local_state._need_insert_cache)); @@ -134,24 +262,31 @@ Status CacheSourceOperatorX::get_block_impl(RuntimeState* state, Block* block, b local_state._global_cache->insert(local_state._cache_key, local_state._version, local_state._local_cache_blocks, local_state._slot_orders, - local_state._current_query_cache_bytes); + local_state._current_query_cache_bytes, + local_state._insert_delta_count); local_state._local_cache_blocks.clear(); + // Latch off so the commit is exactly-once by construction: + // a spurious re-poll after eos (no in-tree driver does + // this today) would otherwise re-publish the now-cleared, + // EMPTY block set over the entry just inserted, and later + // exact hits would silently serve an empty result. This + // is the only source operator whose eos path carries a + // global side effect, so it must not rely on the pull + // protocol never poking it again. + local_state._need_insert_cache = false; } } - }); + }; - std::unique_ptr output_block; - int child_idx = 0; - RETURN_IF_ERROR(local_state._shared_state->data_queue.get_block_from_queue(&output_block, - &child_idx)); - // Here, check the value of `_has_data(state)` again after `data_queue.is_all_finish()` is TRUE - // as there may be one or more blocks when `data_queue.is_all_finish()` is TRUE. - *eos = !_has_data(state) && local_state._shared_state->data_queue.is_all_finish(); + auto queue_block = DORIS_TRY(local_state._shared_state->data_queue.get_block_from_queue()); + *eos = queue_block.eos; - if (!output_block) { + if (!queue_block.block) { + commit_entry_if_finished(); return Status::OK(); } + auto& output_block = queue_block.block; if (local_state._need_insert_cache) { if (need_clone_empty) { *block = output_block->clone_empty(); @@ -160,42 +295,14 @@ Status CacheSourceOperatorX::get_block_impl(RuntimeState* state, Block* block, b auto& mutable_block = scoped_mutable_block.mutable_block(); RETURN_IF_ERROR(mutable_block.merge(*output_block)); scoped_mutable_block.restore(); - local_state._current_query_cache_rows += output_block->rows(); - auto mem_consume = output_block->allocated_bytes(); - local_state._current_query_cache_bytes += mem_consume; - - if (_cache_param.entry_max_bytes < local_state._current_query_cache_bytes || - _cache_param.entry_max_rows < local_state._current_query_cache_rows) { - // over the max bytes, pass through the data, no need to do cache - local_state._local_cache_blocks.clear(); - local_state._need_insert_cache = false; - } else { + if (local_state._account_write_back(output_block->rows(), + output_block->allocated_bytes())) { local_state._local_cache_blocks.emplace_back(std::move(output_block)); } } else { *block = std::move(*output_block); } - } else { - if (local_state._hit_cache_pos < local_state._hit_cache_results->size()) { - const auto& hit_cache_block = - local_state._hit_cache_results->at(local_state._hit_cache_pos++); - if (need_clone_empty) { - *block = hit_cache_block->clone_empty(); - } - ScopedMutableBlock scoped_mutable_block(block); - auto& mutable_block = scoped_mutable_block.mutable_block(); - RETURN_IF_ERROR(mutable_block.merge(*hit_cache_block)); - scoped_mutable_block.restore(); - if (!local_state._hit_cache_column_orders.empty()) { - auto datas = block->get_columns_with_type_and_name(); - block->clear(); - for (auto loc : local_state._hit_cache_column_orders) { - block->insert(datas[loc]); - } - } - } else { - *eos = true; - } + commit_entry_if_finished(); } local_state.reached_limit(block, eos); diff --git a/be/src/exec/operator/cache_source_operator.h b/be/src/exec/operator/cache_source_operator.h index 3a68bd337fc5b7..dd82b29a65e8e0 100644 --- a/be/src/exec/operator/cache_source_operator.h +++ b/be/src/exec/operator/cache_source_operator.h @@ -48,6 +48,20 @@ class CacheSourceLocalState final : public PipelineXLocalState; + // Account `rows`/`bytes` against the entry limits. Once the limits are + // exceeded, drop the pending write-back blocks and stop caching (the data + // itself still passes through to the parent). Returns whether the entry is + // still cacheable. + bool _account_write_back(int64_t rows, int64_t bytes); + // Snapshot `block` (already reordered to this query's slot orders) into + // the write-back set as a zero-copy view sharing its COW columns. Used in + // INCREMENTAL mode for the cached blocks, whose columns belong to the + // entry the decision pins: the write-back set then costs no extra memory + // until QueryCache::insert() performs the single cache-owned + // materialization, so a replacement peaks at old entry + new entry + // instead of three full payloads. + Status _append_block_for_write_back(const Block& block); + QueryCache* _global_cache = QueryCache::instance(); std::string _cache_key {}; @@ -58,7 +72,18 @@ class CacheSourceLocalState final : public PipelineXLocalState _cache_decision; + // Shortcut for _cache_decision->mode == INCREMENTAL: after the cached + // blocks are emitted, the delta partial result is drained from the data + // queue, and both are written back as the merged entry. + bool _is_incremental = false; + // delta_count to write back: 0 for a full recompute, cached + 1 for an + // incremental merge (drives periodic compaction, see QueryCacheRuntime). + int64_t _insert_delta_count = 0; + std::vector* _hit_cache_results = nullptr; std::vector _hit_cache_column_orders; int _hit_cache_pos = 0; @@ -68,8 +93,11 @@ class CacheSourceOperatorX final : public OperatorX { public: using Base = OperatorX; CacheSourceOperatorX(ObjectPool* pool, int plan_node_id, int operator_id, - const TQueryCacheParam& cache_param) - : Base(pool, plan_node_id, operator_id), _cache_param(cache_param) { + const TQueryCacheParam& cache_param, + std::shared_ptr query_cache_runtime) + : Base(pool, plan_node_id, operator_id), + _cache_param(cache_param), + _query_cache_runtime(std::move(query_cache_runtime)) { _op_name = "CACHE_SOURCE_OPERATOR"; }; @@ -90,10 +118,7 @@ class CacheSourceOperatorX final : public OperatorX { private: TQueryCacheParam _cache_param; - bool _has_data(RuntimeState* state) const { - auto& local_state = get_local_state(state); - return local_state._shared_state->data_queue.remaining_has_data(); - } + std::shared_ptr _query_cache_runtime; friend class CacheSourceLocalState; }; diff --git a/be/src/exec/operator/data_queue.cpp b/be/src/exec/operator/data_queue.cpp index 752e972c3e642f..feed0bb79fa265 100644 --- a/be/src/exec/operator/data_queue.cpp +++ b/be/src/exec/operator/data_queue.cpp @@ -25,6 +25,7 @@ #include "common/thread_safety_annotations.h" #include "core/block/block.h" #include "exec/pipeline/dependency.h" +#include "fmt/format.h" namespace doris { @@ -97,6 +98,10 @@ bool DataQueue::has_more_data() const { return _cur_blocks_total_nums.load() > 0; } +std::string DataQueue::debug_string() const { + return fmt::format("(is_all_finish = {}, has_data = {})", is_all_finish(), has_more_data()); +} + void DataQueue::set_source_dependency(std::shared_ptr source_dependency) NO_THREAD_SAFETY_ANALYSIS { _source_dependency = std::move(source_dependency); @@ -134,13 +139,16 @@ std::unique_ptr DataQueue::get_free_block(int child_idx) { return Block::create_unique(); } -void DataQueue::push_free_block(std::unique_ptr block, int child_idx) { - DCHECK(block->rows() == 0); +void DataQueue::push_free_block(DataQueueBlock&& queue_block) { + if (!queue_block.block) { + return; + } + DCHECK(queue_block.block->rows() == 0); if (!_is_low_memory_mode) { - auto& sub = *_sub_queues[child_idx]; + auto& sub = *_sub_queues[queue_block.child_idx]; LockGuard l(sub.free_lock); - sub.free_blocks.emplace_back(std::move(block)); + sub.free_blocks.emplace_back(std::move(queue_block.block)); } } @@ -154,73 +162,75 @@ void DataQueue::clear_free_blocks() { void DataQueue::terminate() { for (int i = 0; i < _child_count; ++i) { - set_finish(i); + mark_finish(i); _sub_queues[i]->clear_blocks(); } + _cur_blocks_total_nums = 0; clear_free_blocks(); + set_source_always_ready(); } -//check which queue have data, and save the idx in _flag_queue_idx, -//so next loop, will check the record idx + 1 first -//maybe it's useful with many queue, others maybe always 0 -bool DataQueue::remaining_has_data() { - int count = _child_count; - while (--count >= 0) { - _flag_queue_idx++; - if (_flag_queue_idx == _child_count) { - _flag_queue_idx = 0; - } - if (_sub_queues[_flag_queue_idx]->blocks_in_queue.load() > 0) { - return true; +Result DataQueue::get_block_from_queue() { + DataQueueBlock result; + const int start_idx = (_flag_queue_idx + 1) % _child_count; + for (int offset = 0; offset < _child_count; ++offset) { + const int idx = (start_idx + offset) % _child_count; + if (_sub_queues[idx]->blocks_in_queue.load() == 0) { + continue; } - } - return false; -} - -//the _flag_queue_idx indicate which queue has data, and in check can_read -//will be set idx in remaining_has_data function -Status DataQueue::get_block_from_queue(std::unique_ptr* output_block, int* child_idx) { - const int idx = _flag_queue_idx; - auto& sub = *_sub_queues[idx]; - sub.try_pop(output_block); - if (*output_block) { - if (child_idx) { - *child_idx = idx; + auto& sub = *_sub_queues[idx]; + sub.try_pop(&result.block); + if (!result.block) { + continue; } + result.child_idx = idx; + _flag_queue_idx = idx; auto old_total = _cur_blocks_total_nums.fetch_sub(1); if (old_total == 1) { set_source_block(); } + break; } - return Status::OK(); + + // A producer enqueues its final block before marking the child finished. Observe completion + // first, then recheck queued data to avoid reporting EOS while the final block is still queued. + result.eos = is_all_finish() && !has_more_data(); + return result; } -Status DataQueue::push_block(std::unique_ptr block, int child_idx) { - if (!block) { +Status DataQueue::push_block(std::unique_ptr block, int child_idx, bool eos) { + DCHECK(block || eos); + if (!block && !eos) { return Status::OK(); } - auto& sub = *_sub_queues[child_idx]; - // total_counter is incremented inside try_push under queue_lock, only when the - // block is actually enqueued. This ensures get_block_from_queue() always observes - // _cur_blocks_total_nums >= 1 when it successfully pops a block, with no risk of - // underflow or the need for a rollback on failure. - if (!sub.try_push(std::move(block), _cur_blocks_total_nums)) { - return Status::EndOfFile("SubQueue already finished"); + + if (block) { + auto& sub = *_sub_queues[child_idx]; + // total_counter is incremented inside try_push under queue_lock, only when the + // block is actually enqueued. This ensures get_block_from_queue() always observes + // _cur_blocks_total_nums >= 1 when it successfully pops a block, with no risk of + // underflow or the need for a rollback on failure. + if (!sub.try_push(std::move(block), _cur_blocks_total_nums)) { + return Status::EndOfFile("SubQueue already finished"); + } + } + + if (eos) { + mark_finish(child_idx); } set_source_ready(); return Status::OK(); } -void DataQueue::set_finish(int child_idx) { +void DataQueue::mark_finish(int child_idx) { auto& sub = *_sub_queues[child_idx]; if (!sub.mark_finished(_un_finished_counter, _is_all_finished)) { return; } - set_source_ready(); } -bool DataQueue::is_all_finish() { +bool DataQueue::is_all_finish() const { return _is_all_finished; } @@ -231,6 +241,13 @@ void DataQueue::set_source_ready() { } } +void DataQueue::set_source_always_ready() { + LockGuard lc(_source_lock); + if (_source_dependency) { + _source_dependency->set_always_ready(); + } +} + void DataQueue::set_source_block() { // Re-check under _source_lock to avoid blocking the source when a concurrent push // has already added new blocks (or all children have finished) since we observed diff --git a/be/src/exec/operator/data_queue.h b/be/src/exec/operator/data_queue.h index 1443e2ed5205bc..49a3bcaf00d8c4 100644 --- a/be/src/exec/operator/data_queue.h +++ b/be/src/exec/operator/data_queue.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "common/status.h" @@ -30,6 +31,15 @@ namespace doris { class Dependency; +struct DataQueueBlock { + std::unique_ptr block; + bool eos = false; + +private: + int child_idx = 0; + friend class DataQueue; +}; + // Per child sub-queue. Groups all parallel state so that the lock/field // relationship is explicit and can be checked by clang -Wthread-safety. struct SubQueue { @@ -46,7 +56,7 @@ struct SubQueue { AnnotatedMutex free_lock; std::deque> free_blocks GUARDED_BY(free_lock); - // blocks_in_queue is readable from lock-free fast paths (remaining_has_data), + // blocks_in_queue is readable from lock-free fast paths (get_block_from_queue), // so it remains atomic and is intentionally not GUARDED_BY. std::atomic_uint32_t blocks_in_queue {0}; @@ -82,18 +92,13 @@ class DataQueue { DataQueue(int child_count = 1); ~DataQueue() = default; - Status get_block_from_queue(std::unique_ptr* block, int* child_idx = nullptr); - Status push_block(std::unique_ptr block, int child_idx = 0); + Result get_block_from_queue(); + Status push_block(std::unique_ptr block, int child_idx, bool eos); std::unique_ptr get_free_block(int child_idx = 0); - void push_free_block(std::unique_ptr output_block, int child_idx = 0); + void push_free_block(DataQueueBlock&& queue_block); - void set_finish(int child_idx = 0); - bool is_all_finish(); - - // This function is not thread safe, should be called in Operator::get_block() - bool remaining_has_data(); - bool has_more_data() const; + std::string debug_string() const; void set_source_dependency(std::shared_ptr source_dependency) NO_THREAD_SAFETY_ANALYSIS; @@ -104,8 +109,12 @@ class DataQueue { void terminate(); private: + bool is_all_finish() const; + bool has_more_data() const; void clear_free_blocks(); + void mark_finish(int child_idx); void set_source_ready(); + void set_source_always_ready(); void set_source_block(); std::vector> _sub_queues; @@ -117,7 +126,7 @@ class DataQueue { std::atomic_uint32_t _cur_blocks_total_nums = 0; //this will be indicate which queue has data, it's useful when have many queues - std::atomic_int _flag_queue_idx = 0; + int _flag_queue_idx = 0; // only used by streaming agg source operator std::atomic_bool _is_low_memory_mode = false; diff --git a/be/src/exec/operator/group_commit_block_sink_operator.cpp b/be/src/exec/operator/group_commit_block_sink_operator.cpp index 66cf4aaee06094..2119cf4093532b 100644 --- a/be/src/exec/operator/group_commit_block_sink_operator.cpp +++ b/be/src/exec/operator/group_commit_block_sink_operator.cpp @@ -76,7 +76,10 @@ Status GroupCommitBlockSinkLocalState::open(RuntimeState* state) { "CreateGroupCommitPlanDependency", true); _put_block_dependency = Dependency::create_shared(_parent->operator_id(), _parent->node_id(), "GroupCommitPutBlockDependency", true); - [[maybe_unused]] auto st = _initialize_load_queue(); + auto st = _initialize_load_queue(); + if (st.is()) { + return st; + } return Status::OK(); } @@ -86,8 +89,8 @@ Status GroupCommitBlockSinkLocalState::_initialize_load_queue() { if (_state->exec_env()->wal_mgr()->is_running()) { RETURN_IF_ERROR(_state->exec_env()->group_commit_mgr()->get_first_block_load_queue( p._db_id, p._table_id, p._base_schema_version, p._schema->indexes().size(), - p._load_id, _load_block_queue, _state->be_exec_version(), _create_plan_dependency, - _put_block_dependency)); + p._load_id, _load_block_queue, _state->be_exec_version(), _group_commit_mode, + _create_plan_dependency, _put_block_dependency)); _state->set_import_label(_load_block_queue->label); _state->set_wal_id(_load_block_queue->txn_id); // wal_id is txn_id return Status::OK(); diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index a5ea092ecc0bf7..50d41d3dcc979a 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -705,6 +705,15 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { // And the user send a query like select userid,count(*) from base table group by userid. // then the storage layer do not need do aggregation, it could just return the partial agg data, because the compute layer will do aggregation. // PreAgg OFF: The storage layer must complete pre-aggregation and return fully aggregated data. (Slow data reading) + // Query cache incremental merge needs no special case here: the read + // sources captured in prepare() already cover exactly the delta versions + // (cached_version, current_version], and the parallel scanner builder + // consumes the captured read sources as they are. The delta is NOT small + // by construction (an entry can sit dormant for arbitrarily many versions + // before it is reused; the merge-count threshold bounds prior write-backs, + // not idle accumulation), so a large suffix is split by rowset/segment + // rows like any full scan, while a small delta naturally collapses to a + // single scanner through min_rows_per_scanner. if (enable_parallel_scan && !p._should_run_serial && p._push_down_agg_type == TPushAggOp::NONE && (_storage_no_merge() || p._olap_scan_node.is_preaggregation) @@ -981,7 +990,31 @@ Status OlapScanLocalState::prepare(RuntimeState* state) { } } + const bool cache_incremental = + _query_cache_decision != nullptr && + _query_cache_decision->mode == QueryCacheInstanceDecision::Mode::INCREMENTAL; for (size_t i = 0; i < _scan_ranges.size(); i++) { + if (cache_incremental) { + // Scan only the delta rowsets in (cached_version, current_version]. + // The read source was already captured (and verified to contain no + // delete predicate) when the cache decision was made, so a capture + // failure cannot surface here, where the cache source operator has + // already committed to emitting the cached blocks. take() returning + // null would mean the FE invariant "one instance per tablet set, no + // shared scan under query cache" was broken (someone else consumed + // this tablet's read source); failing fast is the only safe answer, + // because a silent full-scan fallback would emit the data twice + // (cached blocks + full scan). + auto delta_source = + _query_cache_decision->take_delta_read_source(_tablets[i].tablet->tablet_id()); + if (delta_source == nullptr) { + return Status::InternalError( + "query cache incremental read source is absent, tablet_id={}", + _tablets[i].tablet->tablet_id()); + } + _read_sources[i] = std::move(*delta_source); + continue; + } _read_sources[i] = DORIS_TRY(_tablets[i].tablet->capture_read_source( {0, _tablets[i].version}, {.skip_missing_versions = _state->skip_missing_version(), @@ -1049,28 +1082,29 @@ TOlapScanNode& OlapScanLocalState::olap_scan_node() const { void OlapScanLocalState::set_scan_ranges(RuntimeState* state, const std::vector& scan_ranges) { - const auto& cache_param = _parent->cast()._cache_param; - bool hit_cache = false; - // read binlog scan should not participate in query cache. - if (olap_scan_node().__isset.read_row_binlog && olap_scan_node().read_row_binlog) { - hit_cache = false; - } else if (!cache_param.digest.empty() && !cache_param.force_refresh_query_cache) { - std::string cache_key; - int64_t version = 0; - auto status = QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version); - if (!status.ok()) { - throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, status.msg()); + auto& parent = _parent->cast(); + // Consume the per-instance cache decision. It is made exactly once and + // shared with the cache source operator of this fragment, so the two + // operators always agree: the scan skips scanning if and only if the cache + // source emits the cached blocks (HIT), and scans only the delta rowsets if + // and only if the cache source merges them with the cached blocks + // (INCREMENTAL). The decision also pins the cache entry, so it cannot be + // evicted between the two operators' init. + // Note: an invalid cache key (e.g. FE could not align this instance to one + // partition, so tablets carry different versions) now degrades to an + // uncached full scan instead of failing the query. + if (parent._query_cache_runtime != nullptr) { + _query_cache_decision = parent._query_cache_runtime->get_or_make_decision(scan_ranges); + if (_query_cache_decision->mode == QueryCacheInstanceDecision::Mode::HIT) { + // Exact hit: the cache source emits the entry, nothing to scan. + return; } - doris::QueryCacheHandle handle; - hit_cache = QueryCache::instance()->lookup(cache_key, version, &handle); } - if (!hit_cache) { - for (auto& scan_range : scan_ranges) { - DCHECK(scan_range.scan_range.__isset.palo_scan_range); - _scan_ranges.emplace_back(new TPaloScanRange(scan_range.scan_range.palo_scan_range)); - COUNTER_UPDATE(_tablet_counter, 1); - } + for (auto& scan_range : scan_ranges) { + DCHECK(scan_range.scan_range.__isset.palo_scan_range); + _scan_ranges.emplace_back(new TPaloScanRange(scan_range.scan_range.palo_scan_range)); + COUNTER_UPDATE(_tablet_counter, 1); } } @@ -1263,10 +1297,12 @@ Status OlapScanLocalState::_build_key_ranges_and_filters() { OlapScanOperatorX::OlapScanOperatorX(ObjectPool* pool, const TPlanNode& tnode, int operator_id, const DescriptorTbl& descs, int parallel_tasks, - const TQueryCacheParam& param) + const TQueryCacheParam& param, + std::shared_ptr query_cache_runtime) : ScanOperatorX(pool, tnode, operator_id, descs, parallel_tasks), _olap_scan_node(tnode.olap_scan_node), - _cache_param(param) { + _cache_param(param), + _query_cache_runtime(std::move(query_cache_runtime)) { _output_tuple_id = tnode.olap_scan_node.tuple_id; if (_olap_scan_node.__isset.sort_info && _olap_scan_node.__isset.sort_limit) { DORIS_CHECK(_limit < 0); diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 4e8bab717c6bfa..1d2d2015039d6a 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -33,6 +33,8 @@ namespace doris { class OlapScanner; +class QueryCacheRuntime; +struct QueryCacheInstanceDecision; } // namespace doris namespace doris { @@ -343,6 +345,12 @@ class OlapScanLocalState final : public ScanLocalState { std::vector _tablets; std::vector _read_sources; + // The per-instance query cache decision shared with the cache source + // operator of the same fragment. Null when the query cache is disabled. + // HIT: leave _scan_ranges empty so nothing is scanned; INCREMENTAL: scan + // only the pre-captured delta read sources in (cached, current] version. + std::shared_ptr _query_cache_decision; + std::map _slot_id_to_virtual_column_expr; // ---- Runtime-filter partition pruning ---- @@ -358,7 +366,8 @@ class OlapScanOperatorX final : public ScanOperatorX { public: OlapScanOperatorX(ObjectPool* pool, const TPlanNode& tnode, int operator_id, const DescriptorTbl& descs, int parallel_tasks, - const TQueryCacheParam& cache_param); + const TQueryCacheParam& cache_param, + std::shared_ptr query_cache_runtime = nullptr); Status prepare(RuntimeState* state) override; @@ -374,6 +383,10 @@ class OlapScanOperatorX final : public ScanOperatorX { friend class OlapScanLocalState; TOlapScanNode _olap_scan_node; TQueryCacheParam _cache_param; + // Shared with the cache source operator of the same fragment so both + // consume the same per-instance cache decision (see QueryCacheRuntime). + // Null when the query cache is disabled. + std::shared_ptr _query_cache_runtime; TabletSchemaSPtr _tablet_schema; }; diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index a9e92d08e08613..bc0c56dbfe75fb 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -71,13 +71,19 @@ bool ScanLocalState::should_run_serial() const { return _parent->cast()._should_run_serial; } -Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* state, - int& arrived_rf_num) { +Status ScanLocalStateBase::update_late_arrival_runtime_filter( + RuntimeState* state, int applied_rf_num, int& arrived_rf_num, + VExprContextSPtrs& arrived_conjuncts) { // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads LockGuard lock(_conjuncts_lock); + arrived_conjuncts.clear(); size_t conjuncts_before = _conjuncts.size(); RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter(state, _parent->row_descriptor(), arrived_rf_num, _conjuncts)); + if (_conjuncts.size() > conjuncts_before) { + VExprContextSPtrs appended(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); + _late_arrival_conjunct_batches.emplace_back(arrived_rf_num, std::move(appended)); + } if (state->enable_adjust_conjunct_order_by_cost()) { std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) { return a->execute_cost() < b->execute_cost(); @@ -91,6 +97,16 @@ Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* stat if (_conjuncts.size() > conjuncts_before) { RETURN_IF_ERROR(_on_runtime_filter_update()); } + for (const auto& [batch_arrived_rf_num, batch] : _late_arrival_conjunct_batches) { + if (batch_arrived_rf_num <= applied_rf_num) { + continue; + } + for (const auto& conjunct : batch) { + VExprContextSPtr cloned; + RETURN_IF_ERROR(conjunct->clone(state, cloned)); + arrived_conjuncts.push_back(std::move(cloned)); + } + } return Status::OK(); } @@ -1046,6 +1062,12 @@ TPushAggOp::type ScanLocalState::get_push_down_agg_type() { return _parent->cast()._push_down_agg_type; } +template +const std::optional>& ScanLocalState::get_push_down_count_slot_ids() + const { + return _parent->cast()._push_down_count_slot_ids; +} + template int64_t ScanLocalState::limit_per_scanner() { return _parent->cast()._limit_per_scanner; @@ -1231,6 +1253,9 @@ Status ScanOperatorX::init(const TPlanNode& tnode, RuntimeState* } else { _push_down_agg_type = TPushAggOp::type::NONE; } + if (tnode.__isset.push_down_count_slot_ids) { + _push_down_count_slot_ids = tnode.push_down_count_slot_ids; + } if (tnode.__isset.topn_filter_source_node_ids) { _topn_filter_source_node_ids = tnode.topn_filter_source_node_ids; diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h index 8b12ccf0bc1195..e955840f1311b1 100644 --- a/be/src/exec/operator/scan_operator.h +++ b/be/src/exec/operator/scan_operator.h @@ -18,8 +18,11 @@ #pragma once #include +#include #include #include +#include +#include #include "common/status.h" #include "common/thread_safety_annotations.h" @@ -74,6 +77,7 @@ class ScanLocalStateBase : public PipelineXLocalState<> { virtual void set_scan_ranges(RuntimeState* state, const std::vector& scan_ranges) = 0; virtual TPushAggOp::type get_push_down_agg_type() = 0; + virtual const std::optional>& get_push_down_count_slot_ids() const = 0; // If scan operator is serial operator(like topn), its real parallelism is 1. // Otherwise, its real parallelism is query_parallel_instance_num. @@ -94,7 +98,9 @@ class ScanLocalStateBase : public PipelineXLocalState<> { uint64_t get_condition_cache_digest() const { return _condition_cache_digest; } - Status update_late_arrival_runtime_filter(RuntimeState* state, int& arrived_rf_num); + Status update_late_arrival_runtime_filter(RuntimeState* state, int applied_rf_num, + int& arrived_rf_num, + VExprContextSPtrs& arrived_conjuncts); Status clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjuncts); @@ -140,6 +146,9 @@ class ScanLocalStateBase : public PipelineXLocalState<> { AnnotatedMutex _conjuncts_lock; RuntimeFilterConsumerHelper _helper; + // Preserve append identity independently of the cost-sorted operator snapshot. Every scanner + // needs the exact RF contexts added since its own applied count. + std::vector> _late_arrival_conjunct_batches; // magic number as seed to generate hash value for condition cache uint64_t _condition_cache_digest = 0; // condition cache filter stats @@ -252,6 +261,7 @@ class ScanLocalState : public ScanLocalStateBase { const std::vector& scan_ranges) override {} TPushAggOp::type get_push_down_agg_type() override; + const std::optional>& get_push_down_count_slot_ids() const override; std::vector execution_dependencies() override { if (_filter_dependencies.empty()) { @@ -452,6 +462,16 @@ class ScanOperatorX : public OperatorX { TPushAggOp::type _push_down_agg_type; + // Semantic arguments of a pushed-down COUNT. This is deliberately optional because absence + // and an empty list have different meanings during a BE-first rolling upgrade: + // + // - nullopt: an old FE did not send the field, so the new BE must use the normal scan; + // - empty: the new FE explicitly planned COUNT(*)/COUNT(1); + // - non-empty: the new FE explicitly planned COUNT(col). + // + // Treating nullopt as empty would silently reinterpret an old plan as COUNT(*). + std::optional> _push_down_count_slot_ids; + // Record the value of the aggregate function 'count' from doris's be int64_t _push_down_count = -1; const int _parallel_tasks = 0; diff --git a/be/src/exec/operator/union_sink_operator.cpp b/be/src/exec/operator/union_sink_operator.cpp index b0f7cf619a8ee1..81881fcf5766c4 100644 --- a/be/src/exec/operator/union_sink_operator.cpp +++ b/be/src/exec/operator/union_sink_operator.cpp @@ -100,44 +100,21 @@ Status UnionSinkOperatorX::sink_impl(RuntimeState* state, Block* in_block, bool } SCOPED_TIMER(local_state.exec_time_counter()); COUNTER_UPDATE(local_state.rows_input_counter(), (int64_t)in_block->rows()); - if (local_state._output_block == nullptr) { - local_state._output_block = - local_state._shared_state->data_queue.get_free_block(_cur_child_id); - } - if (_cur_child_id < _get_first_materialized_child_idx()) { //pass_through - if (in_block->rows() > 0) { - local_state._output_block->swap(*in_block); - RETURN_IF_ERROR(local_state._shared_state->data_queue.push_block( - std::move(local_state._output_block), _cur_child_id)); - } + auto output_block = local_state._shared_state->data_queue.get_free_block(_cur_child_id); + if (is_child_passthrough(_cur_child_id)) { //pass_through + output_block->swap(*in_block); } else if (_get_first_materialized_child_idx() != children_count() && _cur_child_id < children_count()) { //need materialized - RETURN_IF_ERROR(materialize_child_block(state, _cur_child_id, in_block, - local_state._output_block.get())); + RETURN_IF_ERROR( + materialize_child_block(state, _cur_child_id, in_block, output_block.get())); } else { return Status::InternalError("maybe can't reach here, execute const expr: {}, {}, {}", _cur_child_id, _get_first_materialized_child_idx(), children_count()); } - if (UNLIKELY(eos)) { - //if _cur_child_id eos, need check to push block - //Now here can't check _output_block rows, even it's row==0, also need push block - //because maybe sink is eos and queue have none data, if not push block - //the source can't can_read again and can't set source finished - if (local_state._output_block) { - RETURN_IF_ERROR(local_state._shared_state->data_queue.push_block( - std::move(local_state._output_block), _cur_child_id)); - } - local_state._shared_state->data_queue.set_finish(_cur_child_id); - return Status::OK(); - } - // not eos and block rows is enough to output,so push block - if (local_state._output_block && (local_state._output_block->rows() >= state->batch_size())) { - RETURN_IF_ERROR(local_state._shared_state->data_queue.push_block( - std::move(local_state._output_block), _cur_child_id)); - } - return Status::OK(); + return local_state._shared_state->data_queue.push_block(std::move(output_block), _cur_child_id, + eos); } } // namespace doris diff --git a/be/src/exec/operator/union_sink_operator.h b/be/src/exec/operator/union_sink_operator.h index 2bf0e7b3e7514f..7f2ec5bbb2f870 100644 --- a/be/src/exec/operator/union_sink_operator.h +++ b/be/src/exec/operator/union_sink_operator.h @@ -51,8 +51,7 @@ class UnionSinkOperatorX; class UnionSinkLocalState final : public PipelineXSinkLocalState { public: ENABLE_FACTORY_CREATOR(UnionSinkLocalState); - UnionSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) - : Base(parent, state), _child_row_idx(0) {} + UnionSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) : Base(parent, state) {} Status init(RuntimeState* state, LocalSinkStateInfo& info) override; Status open(RuntimeState* state) override; friend class UnionSinkOperatorX; @@ -60,8 +59,6 @@ class UnionSinkLocalState final : public PipelineXSinkLocalState _output_block; - /// Const exprs materialized by this node. These exprs don't refer to any children. /// Only materialized by the first fragment instance to avoid duplication. VExprContextSPtrs _const_expr; @@ -69,8 +66,6 @@ class UnionSinkLocalState final : public PipelineXSinkLocalStatedata_queue.is_all_finish(), - _shared_state->data_queue.has_more_data()); + fmt::format_to(debug_string_buffer, ", data_queue: {}", + _shared_state->data_queue.debug_string()); } return fmt::to_string(debug_string_buffer); } Status UnionSourceOperatorX::get_block_impl(RuntimeState* state, Block* block, bool* eos) { auto& local_state = get_local_state(state); - Defer set_eos {[&]() { - // the eos check of union operator is complex, need check all logical if you want modify - // could ref this PR: https://github.com/apache/doris/pull/29677 - // have executing const expr, queue have no data anymore, and child could be closed - if (_child_size == 0 && !local_state._need_read_for_const_expr) { - *eos = true; - } else if (_has_data(state)) { - *eos = false; - } else if (local_state._shared_state->data_queue.is_all_finish()) { - // Here, check the value of `_has_data(state)` again after `data_queue.is_all_finish()` is TRUE - // as there may be one or more blocks when `data_queue.is_all_finish()` is TRUE. - *eos = !_has_data(state); - } else { - *eos = false; - } - }}; - + *eos = false; SCOPED_TIMER(local_state.exec_time_counter()); if (local_state._need_read_for_const_expr) { if (has_more_const(state)) { RETURN_IF_ERROR(get_next_const(state, block)); } local_state._need_read_for_const_expr = has_more_const(state); + if (_child_size == 0 && !local_state._need_read_for_const_expr) { + *eos = true; + } } else if (_child_size != 0) { - std::unique_ptr output_block; - int child_idx = 0; - RETURN_IF_ERROR(local_state._shared_state->data_queue.get_block_from_queue(&output_block, - &child_idx)); - if (!output_block) { + auto queue_block = DORIS_TRY(local_state._shared_state->data_queue.get_block_from_queue()); + *eos = queue_block.eos; + if (!queue_block.block) { return Status::OK(); } - block->swap(*output_block); - output_block->clear_column_data(row_descriptor().num_materialized_slots()); - local_state._shared_state->data_queue.push_free_block(std::move(output_block), child_idx); + block->swap(*queue_block.block); + queue_block.block->clear_column_data(row_descriptor().num_materialized_slots()); + local_state._shared_state->data_queue.push_free_block(std::move(queue_block)); } local_state.reached_limit(block, eos); return Status::OK(); diff --git a/be/src/exec/operator/union_source_operator.h b/be/src/exec/operator/union_source_operator.h index c1006e4669307c..67b405b10022de 100644 --- a/be/src/exec/operator/union_source_operator.h +++ b/be/src/exec/operator/union_source_operator.h @@ -122,13 +122,6 @@ class UnionSourceOperatorX MOCK_REMOVE(final) : public OperatorXdata_queue.remaining_has_data(); - } bool has_more_const(RuntimeState* state) const { auto& local_state = get_local_state(state); return state->per_fragment_instance_idx() == 0 && @@ -140,4 +133,4 @@ class UnionSourceOperatorX MOCK_REMOVE(final) : public OperatorX _const_expr_lists; }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index 5664042dd24e15..def2cf473aea77 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -1519,9 +1519,30 @@ Status PipelineFragmentContext::_create_operator(ObjectPool* pool, const TPlanNo bool fe_with_old_version = false; switch (tnode.node_type) { case TPlanNodeType::OLAP_SCAN_NODE: { + if (enable_query_cache) { + if (_query_cache_runtime == nullptr) { + // The plan tree is built in pre-order and the cache source + // sits above the scan, so the runtime it created must already + // exist here. Running the scan with its own runtime instead + // would silently drop data on a HIT (the scan skips scanning + // while no cache source emits the entry), so a malformed plan + // shape must fail loudly. + return Status::InternalError( + "query cache runtime is absent at the scan node, node_id={}, " + "cache node_id={}", + tnode.node_id, _params.fragment.query_cache_param.node_id); + } + if (tnode.olap_scan_node.__isset.read_row_binlog && + tnode.olap_scan_node.read_row_binlog) { + // Row-binlog scans read a different data stream: they must + // neither serve nor fill the query cache. + _query_cache_runtime->disable_for_binlog_scan(); + } + } op = std::make_shared( pool, tnode, next_operator_id(), descs, _num_instances, - enable_query_cache ? _params.fragment.query_cache_param : TQueryCacheParam {}); + enable_query_cache ? _params.fragment.query_cache_param : TQueryCacheParam {}, + enable_query_cache ? _query_cache_runtime : nullptr); RETURN_IF_ERROR(cur_pipe->add_operator(op, _parallel_instances)); fe_with_old_version = !tnode.__isset.is_serial_operator; break; @@ -1583,8 +1604,13 @@ Status PipelineFragmentContext::_create_operator(ObjectPool* pool, const TPlanNo auto create_query_cache_operator = [&](PipelinePtr& new_pipe) { auto cache_node_id = _params.local_params[0].per_node_scan_ranges.begin()->first; auto cache_source_id = next_operator_id(); + if (_query_cache_runtime == nullptr) { + _query_cache_runtime = + std::make_shared(_params.fragment.query_cache_param); + } op = std::make_shared(pool, cache_node_id, cache_source_id, - _params.fragment.query_cache_param); + _params.fragment.query_cache_param, + _query_cache_runtime); RETURN_IF_ERROR(cur_pipe->add_operator(op, _parallel_instances)); const auto downstream_pipeline_id = cur_pipe->id(); diff --git a/be/src/exec/pipeline/pipeline_fragment_context.h b/be/src/exec/pipeline/pipeline_fragment_context.h index fe1d2a6c065ba0..1a63426738bef8 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.h +++ b/be/src/exec/pipeline/pipeline_fragment_context.h @@ -48,6 +48,7 @@ class ExecEnv; class RuntimeFilterMergeControllerEntity; class TDataSink; class TPipelineFragmentParams; +class QueryCacheRuntime; class Dependency; struct LocalExchangeSharedState; @@ -379,6 +380,12 @@ class PipelineFragmentContext : public TaskExecutionContext { TPipelineFragmentParams _params; int32_t _parallel_instances = 0; + // Query cache context of this fragment, shared by the olap scan operator + // and the cache source operator so both consume the same per-instance + // cache decision (HIT / INCREMENTAL / MISS). Created lazily when the + // fragment carries a query_cache_param. See QueryCacheRuntime. + std::shared_ptr _query_cache_runtime; + std::atomic _need_notify_close = false; // Holds the brpc ClosureGuard for async wait-close during recursive CTE rerun. // When the PFC finishes closing and is destroyed, the shared_ptr destructor fires diff --git a/be/src/exec/scan/access_path_parser.cpp b/be/src/exec/scan/access_path_parser.cpp index b215212b6d861b..1c48b9285d9eda 100644 --- a/be/src/exec/scan/access_path_parser.cpp +++ b/be/src/exec/scan/access_path_parser.cpp @@ -87,10 +87,18 @@ void inherit_schema_metadata(format::ColumnDefinition* column, return; } column->name_mapping = schema_column->name_mapping; + // The presence bit is part of the mapping contract: an explicit empty mapping must remain + // authoritative after access-path pruning instead of enabling current-name fallback. + column->has_name_mapping = schema_column->has_name_mapping; + // Initial defaults describe the logical value of fields absent from older files. Nested + // access-path pruning must retain them just like it retains rename metadata. + column->initial_default_value = schema_column->initial_default_value; + column->initial_default_value_is_base64 = schema_column->initial_default_value_is_base64; } const format::ColumnDefinition* find_schema_child_by_path( - const format::ColumnDefinition* schema_column, const std::string& child_path) { + const format::ColumnDefinition* schema_column, const std::string& child_path, + bool prefer_exact_name_match) { if (schema_column == nullptr) { return nullptr; } @@ -103,15 +111,31 @@ const format::ColumnDefinition* find_schema_child_by_path( }); return child_it == schema_column->children.end() ? nullptr : &*child_it; } - const auto child_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { - if (to_lower(child.name) == to_lower(child_path)) { - return true; - } + if (!prefer_exact_name_match) { + const auto child_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { + if (to_lower(child.name) == to_lower(child_path)) { + return true; + } + return std::ranges::any_of(child.name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(child_path); + }); + }); + return child_it == schema_column->children.end() ? nullptr : &*child_it; + } + // Iceberg can reuse a historical name for a newly added sibling. Current names therefore + // have precedence across the entire struct; an earlier alias must not steal that access path. + const auto exact_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { + return to_lower(child.name) == to_lower(child_path); + }); + if (exact_it != schema_column->children.end()) { + return &*exact_it; + } + const auto alias_it = std::ranges::find_if(schema_column->children, [&](const auto& child) { return std::ranges::any_of(child.name_mapping, [&](const std::string& alias) { return to_lower(alias) == to_lower(child_path); }); }); - return child_it == schema_column->children.end() ? nullptr : &*child_it; + return alias_it == schema_column->children.end() ? nullptr : &*alias_it; } int32_t schema_field_id(const format::ColumnDefinition* schema_column) { @@ -169,7 +193,8 @@ void insert_access_path(AccessPathNode* root, const std::vector& pa Status build_nested_children_from_access_node(format::ColumnDefinition* column, const DataTypePtr& type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column); + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match); // Expand a full complex-column projection into table-schema children when the table format provides // an external/current schema. Without this, `SELECT complex_col` or `SELECT *` leaves @@ -186,7 +211,8 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, // wrapper here: ColumnMapper and TableReader treat MAP children as [key, value]. Status build_all_nested_children_from_schema(format::ColumnDefinition* column, const DataTypePtr& type, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); const auto nested_type = remove_nullable(type); @@ -197,14 +223,16 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, const auto& struct_type = assert_cast(*nested_type); for (size_t field_idx = 0; field_idx < struct_type.get_elements().size(); ++field_idx) { const auto field_name = struct_type.get_element_name(field_idx); - const auto* schema_child = find_schema_child_by_path(schema_column, field_name); + const auto* schema_child = + find_schema_child_by_path(schema_column, field_name, prefer_exact_name_match); auto* child = find_or_add_child( column, schema_field_id_or(schema_child, cast_set(field_idx)), schema_field_name_or(schema_child, field_name), struct_type.get_element(field_idx)); inherit_schema_metadata(child, schema_child); RETURN_IF_ERROR(build_nested_children_from_access_node( - child, child->type, project_all, path + "." + child->name, schema_child)); + child, child->type, project_all, path + "." + child->name, schema_child, + prefer_exact_name_match)); } return Status::OK(); } @@ -217,7 +245,7 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, array_type.get_nested_type()); inherit_schema_metadata(child, element_schema); return build_nested_children_from_access_node(child, child->type, project_all, path + ".*", - element_schema); + element_schema, prefer_exact_name_match); } case TYPE_MAP: { const auto& map_type = assert_cast(*nested_type); @@ -231,12 +259,14 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, map_type.get_key_type()); inherit_schema_metadata(key_child, key_schema); RETURN_IF_ERROR(build_nested_children_from_access_node( - key_child, key_child->type, project_all, path + ".KEYS", key_schema)); + key_child, key_child->type, project_all, path + ".KEYS", key_schema, + prefer_exact_name_match)); auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value", map_type.get_value_type()); inherit_schema_metadata(value_child, value_schema); RETURN_IF_ERROR(build_nested_children_from_access_node( - value_child, value_child->type, project_all, path + ".VALUES", value_schema)); + value_child, value_child->type, project_all, path + ".VALUES", value_schema, + prefer_exact_name_match)); return Status::OK(); } default: @@ -247,7 +277,8 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column, Status build_struct_children_from_access_node(format::ColumnDefinition* column, const DataTypeStruct& struct_type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); for (const auto& [child_path, child_node] : node.children) { // Struct children are resolved by name or schema field id. We do not treat a numeric @@ -262,7 +293,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column, // Prefer the table/schema ColumnDefinition because it carries field ids and aliases. // Fallback to the struct type name only for formats without external schema metadata. - const auto* schema_child = find_schema_child_by_path(schema_column, child_path); + const auto* schema_child = + find_schema_child_by_path(schema_column, child_path, prefer_exact_name_match); int32_t field_id = schema_field_id(schema_child); std::string field_name = schema_child == nullptr ? child_path : schema_child->name; DataTypePtr field_type = schema_child == nullptr ? nullptr : schema_child->type; @@ -288,7 +320,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column, auto* child = find_or_add_child(column, field_id, field_name, field_type); inherit_schema_metadata(child, schema_child); RETURN_IF_ERROR(build_nested_children_from_access_node( - child, child->type, child_node, path + "." + child_path, schema_child)); + child, child->type, child_node, path + "." + child_path, schema_child, + prefer_exact_name_match)); } return Status::OK(); } @@ -296,7 +329,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column, Status build_map_children_from_access_node(format::ColumnDefinition* column, const DataTypeMap& map_type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); AccessPathNode key_node; AccessPathNode value_node; @@ -368,14 +402,16 @@ Status build_map_children_from_access_node(format::ColumnDefinition* column, map_type.get_key_type()); inherit_schema_metadata(key_child, key_schema); RETURN_IF_ERROR(build_nested_children_from_access_node(key_child, key_child->type, key_node, - path + ".KEYS", key_schema)); + path + ".KEYS", key_schema, + prefer_exact_name_match)); } if (need_value) { auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value", map_type.get_value_type()); inherit_schema_metadata(value_child, value_schema); RETURN_IF_ERROR(build_nested_children_from_access_node( - value_child, value_child->type, value_node, path + ".VALUES", value_schema)); + value_child, value_child->type, value_node, path + ".VALUES", value_schema, + prefer_exact_name_match)); } return Status::OK(); } @@ -383,18 +419,20 @@ Status build_map_children_from_access_node(format::ColumnDefinition* column, Status build_nested_children_from_access_node(format::ColumnDefinition* column, const DataTypePtr& type, const AccessPathNode& node, const std::string& path, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); if (node.project_all || node.children.empty()) { - return build_all_nested_children_from_schema(column, type, path, schema_column); + return build_all_nested_children_from_schema(column, type, path, schema_column, + prefer_exact_name_match); } const auto nested_type = remove_nullable(type); switch (nested_type->get_primitive_type()) { case TYPE_STRUCT: return build_struct_children_from_access_node( - column, assert_cast(*nested_type), node, path, - schema_column); + column, assert_cast(*nested_type), node, path, schema_column, + prefer_exact_name_match); case TYPE_ARRAY: { if (node.children.size() != 1 || !node.children.contains("*")) { return Status::NotSupported( @@ -409,11 +447,13 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, array_type.get_nested_type()); inherit_schema_metadata(child, element_schema); return build_nested_children_from_access_node(child, child->type, node.children.at("*"), - path + ".*", element_schema); + path + ".*", element_schema, + prefer_exact_name_match); } case TYPE_MAP: return build_map_children_from_access_node( - column, assert_cast(*nested_type), node, path, schema_column); + column, assert_cast(*nested_type), node, path, schema_column, + prefer_exact_name_match); default: return Status::NotSupported("AccessPathParser does not support access path {} for slot {}", path, column->name); @@ -424,7 +464,8 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column, Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, const std::vector& access_paths, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); if (is_scanner_materialized_virtual_column(column->name)) { return Status::OK(); @@ -465,15 +506,17 @@ Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, } // Recursively build nested children for the column based on the AccessPathNode tree. return build_nested_children_from_access_node(column, column->type, root, column->name, - schema_column); + schema_column, prefer_exact_name_match); } Status AccessPathParser::build_nested_children(format::ColumnDefinition* column, const SlotDescriptor* slot_desc, - const format::ColumnDefinition* schema_column) { + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match) { DORIS_CHECK(column != nullptr); DORIS_CHECK(slot_desc != nullptr); - return build_nested_children(column, slot_desc->all_access_paths(), schema_column); + return build_nested_children(column, slot_desc->all_access_paths(), schema_column, + prefer_exact_name_match); } } // namespace doris diff --git a/be/src/exec/scan/access_path_parser.h b/be/src/exec/scan/access_path_parser.h index 1aa4c5b89d492a..0be785a33906e8 100644 --- a/be/src/exec/scan/access_path_parser.h +++ b/be/src/exec/scan/access_path_parser.h @@ -31,11 +31,13 @@ class AccessPathParser { public: static Status build_nested_children(format::ColumnDefinition* column, const SlotDescriptor* slot_desc, - const format::ColumnDefinition* schema_column); + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match = true); static Status build_nested_children(format::ColumnDefinition* column, const std::vector& access_paths, - const format::ColumnDefinition* schema_column); + const format::ColumnDefinition* schema_column, + bool prefer_exact_name_match = true); }; } // namespace doris diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 343c87edc51196..8851d9b2c05724 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -1942,25 +1942,7 @@ bool FileScanner::_should_enable_condition_cache() { return false; } - // Runtime filters are query-local dynamic predicates. Some ready RF implementations can hash - // their payload into get_digest(), but FileScanner cannot rely on that for all RFs reaching the - // native reader. In particular, ScanLocalState computes _condition_cache_digest during open(), - // while FileScanner may append late-arrival RFs in _process_late_arrival_conjuncts() - // immediately before initializing Parquet/ORC readers. - // - // Reading a weaker cache entry would be safe by itself: if a cached bitmap only represented - // static predicate P, false granules for P are also false for P AND RF. The unsafe part is - // writing. On cache miss, native readers mark survivor granules using all pushed-down - // predicates, including late RFs. Without a read-only cache mode, this would insert a bitmap for - // P AND RF under a digest that only represents P. - // - // Example: - // Q1 static predicate: k = 1, late RF payload: partition_key IN ('2024-02-01') - // Q2 static predicate: k = 1, late RF payload: partition_key IN ('2024-03-01') - // If both scans share the same file/range/digest, reusing Q1's bitmap for Q2 can skip row - // ranges according to the wrong RF payload. Keep RF predicate pushdown enabled for reader-side - // filtering, but do not persist its result in condition cache. - return !_contains_runtime_filter(_conjuncts) && !_contains_runtime_filter(_push_down_conjuncts); + return true; } bool FileScanner::_should_enable_condition_cache_for_load() const { @@ -1990,6 +1972,12 @@ void FileScanner::_init_reader_condition_cache() { _condition_cache = nullptr; _condition_cache_ctx = nullptr; + // _process_late_arrival_conjuncts() runs before the Parquet/ORC reader is initialized. Rebuild + // the key here so a newly arrived cache-safe RF is represented by both its semantics and its + // payload. If any current conjunct cannot produce a reliable digest, this becomes zero and the + // existing safety gate below disables condition cache. + _condition_cache_digest = _current_condition_cache_digest(); + if (!_should_enable_condition_cache() || !_cur_reader) { return; } diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index 87a54243f16c96..f575cdae89263f 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -80,6 +81,10 @@ class FileScanner : public Scanner { const VExprContextSPtrs& TEST_runtime_filter_partition_prune_ctxs() const { return _runtime_filter_partition_prune_ctxs; } + static TPushAggOp::type TEST_effective_push_down_agg_type( + TPushAggOp::type agg_type, const std::optional>& count_slot_ids) { + return _effective_push_down_agg_type(agg_type, count_slot_ids); + } #endif FileScanner(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -334,9 +339,28 @@ class FileScanner : public Scanner { void _update_adaptive_batch_size_before_truncate(const Block& block); void _update_adaptive_batch_size_after_truncate(const Block& block); + static TPushAggOp::type _effective_push_down_agg_type( + TPushAggOp::type agg_type, const std::optional>& count_slot_ids) { + if (agg_type != TPushAggOp::type::COUNT) { + return agg_type; + } + // V1's CountReader receives only the file's total row count and emits that many synthetic + // rows. This is exact for COUNT(*)/COUNT(1), but it has no column metadata for NULL or CAST + // semantics. For example, a 10,000-row file with 9,015 non-null values must return 10,000 + // for COUNT(*) and 9,015 for COUNT(nullable_col); CountReader can produce only the former. + // Therefore a non-empty argument list must use the normal reader. nullopt is an old FE plan + // that predates the argument field; treating it as empty would silently reinterpret unknown + // semantics as COUNT(*). + return count_slot_ids.has_value() && count_slot_ids->empty() ? TPushAggOp::type::COUNT + : TPushAggOp::type::NONE; + } + TPushAggOp::type _get_push_down_agg_type() const { - return _local_state == nullptr ? TPushAggOp::type::NONE - : _local_state->get_push_down_agg_type(); + if (_local_state == nullptr) { + return TPushAggOp::type::NONE; + } + return _effective_push_down_agg_type(_local_state->get_push_down_agg_type(), + _local_state->get_push_down_count_slot_ids()); } // enable the file meta cache only when diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index ef0896867c1a7e..a20946cf095d51 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -48,6 +49,7 @@ #include "exprs/vexpr_context.h" #include "exprs/vslot_ref.h" #include "format/format_common.h" +#include "format/table/iceberg_scan_semantics.h" #include "format_v2/column_mapper.h" #include "format_v2/jni/iceberg_sys_table_reader.h" #include "format_v2/jni/jdbc_reader.h" @@ -65,6 +67,7 @@ #include "io/io_common.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "service/backend_options.h" #include "storage/id_manager.h" @@ -197,13 +200,9 @@ Status rewrite_slot_refs_to_global_index( const auto* slot_ref = assert_cast(expr->get()); const auto global_index_it = slot_id_to_global_index.find(slot_ref->slot_id()); if (global_index_it == slot_id_to_global_index.end()) { - DORIS_CHECK(slot_ref->slot_id() >= 0); - const auto global_index = format::GlobalIndex(cast_set(slot_ref->slot_id())); - *expr = VSlotRef::create_shared(cast_set(global_index.value()), - cast_set(global_index.value()), -1, - slot_ref->data_type(), slot_ref->column_name()); - RETURN_IF_ERROR(expr->get()->prepare(nullptr, RowDescriptor(), nullptr)); - return Status::OK(); + return Status::InternalError( + "Can not resolve source slot id {} to a table global index for column {}", + slot_ref->slot_id(), slot_ref->column_name()); } const auto global_index = global_index_it->second; *expr = VSlotRef::create_shared(cast_set(global_index.value()), @@ -228,7 +227,9 @@ Status rewrite_slot_refs_to_global_index( #ifdef BE_TEST FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile, std::unique_ptr table_reader) - : Scanner(state, profile), _table_reader(std::move(table_reader)) {} + : Scanner(state, profile), + _table_reader(std::move(table_reader)), + _scanner_profile(profile) {} Status FileScannerV2::TEST_validate_scan_range(const TFileScanRangeParams& params, const TFileRangeDesc& range) { @@ -276,6 +277,10 @@ void FileScannerV2::TEST_report_file_cache_profile( bool FileScannerV2::TEST_should_skip_not_found(const Status& status, bool ignore_not_found) { return _should_skip_not_found(status, ignore_not_found); } + +bool FileScannerV2::TEST_should_skip_empty(const Status& status, bool stopped) { + return _should_skip_empty(status, stopped); +} #endif bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFileRangeDesc& range) { @@ -324,24 +329,49 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { RETURN_IF_ERROR(Scanner::init(state, conjuncts)); - _get_block_timer = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerV2GetBlockTime", 1); - _not_found_file_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "NotFoundFileNum", TUnit::UNIT, 1); - _file_counter = - ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT, 1); - _file_read_bytes_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadBytes", TUnit::BYTES, 1); - _file_read_calls_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadCalls", TUnit::UNIT, 1); + _initialize_scanner_residual_conjuncts(); + auto* profile = _local_state->scanner_profile(); + _scanner_profile = profile; + const auto hierarchy = file_scan_profile::ensure_hierarchy(profile); + _scanner_total_timer = hierarchy.scanner; + _io_timer = hierarchy.io; + _init_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2InitTime", + file_scan_profile::SCANNER, 1); + _open_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2OpenTime", + file_scan_profile::SCANNER, 1); + _get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetBlockTime", + file_scan_profile::SCANNER, 1); + _prepare_split_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2PrepareSplitTime", + file_scan_profile::SCANNER, 1); + _get_next_range_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetNextRangeTime", + file_scan_profile::SCANNER, 1); + _close_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2CloseTime", + file_scan_profile::SCANNER, 1); + _empty_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptyFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _not_found_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NotFoundFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNumber", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_read_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadBytes", TUnit::BYTES, + file_scan_profile::IO, 1); + _file_read_calls_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadCalls", TUnit::UNIT, + file_scan_profile::IO, 1); _file_read_time_counter = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileReadTime", 1); - _adaptive_batch_predicted_rows_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchPredictedRows", TUnit::UNIT, 1); - _adaptive_batch_actual_bytes_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchActualBytes", TUnit::BYTES, 1); - _adaptive_batch_probe_count_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchProbeCount", TUnit::UNIT, 1); + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReadTime", file_scan_profile::IO, 1); + _adaptive_batch_predicted_rows_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchPredictedRows", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _adaptive_batch_actual_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchActualBytes", TUnit::BYTES, file_scan_profile::SCANNER, 1); + _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchProbeCount", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _scanner_residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL( + profile, "ScannerResidualFilterTime", file_scan_profile::SCANNER, 1); + _scanner_residual_rows_filtered_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ScannerResidualRowsFiltered", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _refresh_scanner_residual_profile(); + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_init_timer); _file_cache_statistics = std::make_unique(); _file_reader_stats = std::make_unique(); RETURN_IF_ERROR(_init_io_ctx()); @@ -352,6 +382,8 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc } Status FileScannerV2::_open_impl(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_open_timer); RETURN_IF_CANCELLED(state); RETURN_IF_ERROR(Scanner::_open_impl(state)); RETURN_IF_ERROR(_get_next_scan_range(&_first_scan_range)); @@ -365,6 +397,7 @@ Status FileScannerV2::_open_impl(RuntimeState* state) { } Status FileScannerV2::_get_next_scan_range(bool* has_next) { + SCOPED_TIMER(_get_next_range_timer); DORIS_CHECK(has_next != nullptr); RETURN_IF_ERROR(_split_source->get_next(has_next, &_current_range)); if (*has_next) { @@ -374,8 +407,11 @@ Status FileScannerV2::_get_next_scan_range(bool* has_next) { } Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* eof) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_get_block_timer); while (true) { RETURN_IF_CANCELLED(state); + RETURN_IF_ERROR(_sync_table_reader_conjuncts()); if (!_has_prepared_split) { RETURN_IF_ERROR(_prepare_next_split(eof)); if (*eof) { @@ -384,7 +420,6 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } { - SCOPED_TIMER(_get_block_timer); if (_should_run_adaptive_batch_size()) { _table_reader->set_batch_size(_predict_reader_batch_rows()); } @@ -398,6 +433,20 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e *eof = false; continue; } + if (_should_skip_empty(status, _should_stop || _io_ctx->should_stop)) { + // END_OF_FILE here means the reader discovered a valid split with no data while + // opening or probing it, not that the Scanner has exhausted all splits. Examples + // are a zero-byte CSV with an explicit schema and a Doris Native file containing + // only its 12-byte header. Treat it like V1's empty-file path: finish this range, + // discard partial reader state, and let the loop fetch the next split. + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_empty_file_counter, 1); + _state->update_num_finished_scan_range(1); + _has_prepared_split = false; + block->clear_column_data(cast_set(_projected_columns.size())); + *eof = false; + continue; + } RETURN_IF_ERROR(status); } if (*eof) { @@ -411,7 +460,38 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } } +Status FileScannerV2::_filter_output_block(Block* block) { + if (_scanner_residual_conjuncts.empty() || block->rows() == 0) { + return Status::OK(); + } + SCOPED_TIMER(_scanner_residual_filter_timer); + const size_t rows_before_filter = block->rows(); + auto status = VExprContext::filter_block(_scanner_residual_conjuncts, block, block->columns()); + if (!status.ok() && _params != nullptr && + _get_current_format_type() == TFileFormatType::FORMAT_ORC) { + status.prepend("Orc row reader nextBatch failed. reason = "); + } + RETURN_IF_ERROR(status); + const int64_t filtered_rows = cast_set(rows_before_filter - block->rows()); + _counter.num_rows_unselected += filtered_rows; + if (_scanner_residual_rows_filtered_counter != nullptr) { + COUNTER_UPDATE(_scanner_residual_rows_filtered_counter, filtered_rows); + } + return Status::OK(); +} + +size_t FileScannerV2::_last_block_rows_read(const Block& block) const { + const auto& stats = _table_reader->last_materialized_block_stats(); + return stats.has_materialized_input ? stats.rows : block.rows(); +} + +size_t FileScannerV2::_last_block_bytes_read(const Block& block) const { + const auto& stats = _table_reader->last_materialized_block_stats(); + return stats.has_materialized_input ? stats.allocated_bytes : block.allocated_bytes(); +} + Status FileScannerV2::_prepare_next_split(bool* eos) { + SCOPED_TIMER(_prepare_split_timer); while (true) { bool has_next = _first_scan_range; if (!_first_scan_range) { @@ -427,9 +507,12 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { const auto format_type = get_range_format_type(*_params, _current_range); _init_adaptive_batch_size_state(format_type); - if (_should_run_adaptive_batch_size()) { - // JNI readers open eagerly in prepare_split(). Seed the probe size first so readers - // such as Paimon also use it for their first physical read batch. + if (_block_size_predictor != nullptr) { + // JNI readers open eagerly in prepare_split(). Always seed the probe before preparing + // the next split: its metadata-COUNT decision is not available yet, and the state + // exposed by TableReader can still describe the preceding split. Metadata shortcuts + // ignore this batch size, while row-scan fallbacks need it for their first physical + // read batch. _table_reader->set_batch_size(_predict_reader_batch_rows()); } std::map partition_values; @@ -442,6 +525,16 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { _state->update_num_finished_scan_range(1); continue; } + if (_should_skip_empty(status, _should_stop || _io_ctx->should_stop)) { + // Schema discovery can reach EOF before a split becomes prepared. A header-only Native + // file follows this path, while a reader that discovers emptiness on its first + // get_block() follows the symmetric branch in _get_block_impl(). Both paths must + // advance exactly one scan range and preserve later files in the same scan. + RETURN_IF_ERROR(_table_reader->abort_split()); + COUNTER_UPDATE(_empty_file_counter, 1); + _state->update_num_finished_scan_range(1); + continue; + } RETURN_IF_ERROR(status); if (_table_reader->current_split_pruned()) { _state->update_num_finished_scan_range(1); @@ -462,9 +555,25 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { VExprContextSPtrs table_conjuncts; RETURN_IF_ERROR(_build_table_conjuncts(&table_conjuncts)); + std::optional> push_down_count_columns; + const auto& push_down_count_slot_ids = _local_state->get_push_down_count_slot_ids(); + if (push_down_count_slot_ids.has_value()) { + push_down_count_columns.emplace(); + push_down_count_columns->reserve(push_down_count_slot_ids->size()); + for (const auto slot_id : *push_down_count_slot_ids) { + const auto global_index_it = _slot_id_to_global_index.find(slot_id); + if (global_index_it == _slot_id_to_global_index.end()) { + return Status::InternalError( + "Pushed-down COUNT argument is not a projected file scan slot, slot_id={}", + slot_id); + } + push_down_count_columns->push_back(global_index_it->second); + } + } RETURN_IF_ERROR(_table_reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(table_conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = const_cast(_params), .io_ctx = _io_ctx, @@ -472,8 +581,12 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .scanner_profile = _local_state->scanner_profile(), .file_slot_descs = &_file_slot_descs, .push_down_agg_type = _local_state->get_push_down_agg_type(), + .push_down_count_columns = std::move(push_down_count_columns), .condition_cache_digest = _local_state->get_condition_cache_digest(), })); + _table_reader_applied_rf_num = _applied_rf_num; + // RFs collected before TableReader initialization are already present in the full snapshot. + _late_arrival_rf_conjuncts.clear(); return Status::OK(); } @@ -518,19 +631,17 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range, std::map partition_values) { format::FileFormat current_split_format; RETURN_IF_ERROR(_to_file_format(get_range_format_type(*_params, range), ¤t_split_format)); - VExprContextSPtrs conjuncts; - RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts)); VExprContextSPtrs partition_prune_conjuncts; if (_state->query_options().enable_runtime_filter_partition_prune) { RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts)); } RETURN_IF_ERROR(_table_reader->prepare_split({ .partition_values = std::move(partition_values), - .conjuncts = std::move(conjuncts), .partition_prune_conjuncts = std::move(partition_prune_conjuncts), // A metadata COUNT split may span scheduler turns. Do not enter that irreversible // synthetic-row path while a runtime filter can still arrive between batches. .all_runtime_filters_applied = _applied_rf_num == _total_rf_num, + .condition_cache_digest = _current_condition_cache_digest(), .cache = _kv_cache, .current_range = range, .current_split_format = current_split_format, @@ -543,6 +654,14 @@ bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not return ignore_not_found && status.is(); } +bool FileScannerV2::_should_skip_empty(const Status& status, bool stopped) { + // Several readers use END_OF_FILE both for a valid zero-row split and for an interrupted IO. + // For example, DeletionVectorReader returns END_OF_FILE("stop read.") after try_stop() marks + // the shared IOContext. That status must unwind the stopped scanner; counting it as an empty + // file would incorrectly finish the scan range and increment EmptyFileNum. + return !stopped && status.is(); +} + bool FileScannerV2::_should_enable_file_meta_cache() const { return ExecEnv::GetInstance()->file_meta_cache()->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; @@ -634,6 +753,9 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_ .range = &_current_range, .runtime_state = _state, }; + // Field 34 is the rollout boundary for root and nested exact-name precedence. + const bool prefer_exact_name_match = + !_params->__isset.history_schema_info || supports_iceberg_scan_semantics_v1(_params); for (size_t slot_idx = 0; slot_idx < _params->required_slots.size(); ++slot_idx) { const auto& slot_info = _params->required_slots[slot_idx]; @@ -654,7 +776,8 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_ // column's nested children. RETURN_IF_ERROR(AccessPathParser::build_nested_children( &column, it->second, - build_context.schema_column.has_value() ? &*build_context.schema_column : nullptr)); + build_context.schema_column.has_value() ? &*build_context.schema_column : nullptr, + prefer_exact_name_match)); if (is_partition_slot(slot_info, column.name)) { column.is_partition_key = true; _partition_slot_descs.emplace( @@ -703,10 +826,15 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor } Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const { + return _build_table_conjuncts(_conjuncts, conjuncts); +} + +Status FileScannerV2::_build_table_conjuncts(const VExprContextSPtrs& source, + VExprContextSPtrs* conjuncts) const { DORIS_CHECK(conjuncts != nullptr); conjuncts->clear(); - conjuncts->reserve(_conjuncts.size()); - for (const auto& conjunct : _conjuncts) { + conjuncts->reserve(source.size()); + for (const auto& conjunct : source) { VExprSPtr root; RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); RETURN_IF_ERROR(rewrite_slot_refs_to_global_index(&root, _slot_id_to_global_index)); @@ -715,6 +843,68 @@ Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const return Status::OK(); } +size_t FileScannerV2::_safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts) { + for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { + if (!format::TableReader::is_safe_to_pre_execute(conjuncts[conjunct_index])) { + return conjunct_index; + } + } + return conjuncts.size(); +} + +void FileScannerV2::_initialize_scanner_residual_conjuncts() { + _table_reader_owned_conjunct_count = _safe_conjunct_prefix_size(_conjuncts); + // Preserve the entire suffix, not only the unsafe expression. Otherwise a later safe + // predicate could run below Scanner before a stateful/error-preserving ordering barrier. + _scanner_residual_conjuncts.assign( + _conjuncts.begin() + cast_set(_table_reader_owned_conjunct_count), + _conjuncts.end()); + _refresh_scanner_residual_profile(); +} + +void FileScannerV2::_refresh_scanner_residual_profile() { + if (_scanner_profile == nullptr || _scanner_residual_conjuncts.empty()) { + return; + } + std::ostringstream predicates; + predicates << "["; + for (size_t conjunct_index = 0; conjunct_index < _scanner_residual_conjuncts.size(); + ++conjunct_index) { + if (conjunct_index > 0) { + predicates << ", "; + } + predicates << _scanner_residual_conjuncts[conjunct_index]->root()->debug_string(); + } + predicates << "]"; + _scanner_profile->add_info_string("ScannerResidualPredicates", predicates.str()); +} + +Status FileScannerV2::_sync_table_reader_conjuncts() { + if (_table_reader == nullptr) { + return Status::OK(); + } + if (_table_reader_applied_rf_num == _applied_rf_num) { + return Status::OK(); + } + VExprContextSPtrs appended; + RETURN_IF_ERROR(_build_table_conjuncts(_late_arrival_rf_conjuncts, &appended)); + const size_t owned_count = _scanner_residual_conjuncts.empty() + ? _safe_conjunct_prefix_size(_late_arrival_rf_conjuncts) + : 0; + // Preserve existing expression state and append the identity-tracked RF delta. Cost sorting + // may move a late RF ahead of an old stateful predicate in the full scanner snapshot. + RETURN_IF_ERROR(_table_reader->append_conjuncts_with_ownership(appended, owned_count)); + _table_reader_owned_conjunct_count += owned_count; + _scanner_residual_conjuncts.insert( + _scanner_residual_conjuncts.end(), + _late_arrival_rf_conjuncts.begin() + cast_set(owned_count), + _late_arrival_rf_conjuncts.end()); + _refresh_scanner_residual_profile(); + _late_arrival_rf_conjuncts.clear(); + _table_reader_applied_rf_num = _applied_rf_num; + return Status::OK(); +} + TFileFormatType::type FileScannerV2::_get_current_format_type() const { return get_range_format_type(*_params, _current_range); } @@ -812,10 +1002,17 @@ bool FileScannerV2::_should_enable_adaptive_batch_size(TFileFormatType::type for } bool FileScannerV2::_should_run_adaptive_batch_size() const { - // COUNT pushdown emits synthetic rows from file metadata and does not materialize file columns, - // so there is no useful row-width sample to learn from. - return _block_size_predictor != nullptr && - _local_state->get_push_down_agg_type() != TPushAggOp::type::COUNT; + DORIS_CHECK(_table_reader != nullptr); + return _should_run_adaptive_batch_size(_block_size_predictor != nullptr, + _table_reader->current_split_uses_metadata_count()); +} + +bool FileScannerV2::_should_run_adaptive_batch_size(bool predictor_initialized, + bool current_split_uses_metadata_count) { + // Metadata COUNT emits synthetic rows and has no physical row width to learn from. A raw COUNT + // opcode is not sufficient here: unsupported argument counts, mappings, filters, or deletes + // make TableReader fall back to materializing normal rows, which still need adaptive batching. + return predictor_initialized && !current_split_uses_metadata_count; } size_t FileScannerV2::_predict_reader_batch_rows() { @@ -831,20 +1028,24 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) { if (!_should_run_adaptive_batch_size()) { return; } - COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast(block.bytes())); - if (block.rows() == 0) { + const auto& stats = _table_reader->last_materialized_block_stats(); + const size_t rows = stats.has_materialized_input ? stats.rows : block.rows(); + const size_t bytes = stats.has_materialized_input ? stats.bytes : block.bytes(); + COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast(bytes)); + if (rows == 0) { return; } - // The sample is taken after TableReader has finalized file-local columns to table columns. - // This matches the memory shape seen by upstream operators and catches very wide nested - // columns, such as map/string payloads, after the first probe batch. + // Residual predicates run after wide table columns are materialized. Learn from that pre-filter + // shape so selective predicates cannot make the next reader batch dangerously large. if (!_block_size_predictor->has_history()) { COUNTER_UPDATE(_adaptive_batch_probe_count_counter, 1); } - _block_size_predictor->update(block); + _block_size_predictor->update(rows, bytes); } Status FileScannerV2::close(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_close_timer); if (!_try_close()) { return Status::OK(); } @@ -984,14 +1185,31 @@ void FileScannerV2::_collect_profile_before_close() { Scanner::_collect_profile_before_close(); if (config::enable_file_cache && _state->query_options().enable_file_cache && _profile != nullptr) { - _report_file_cache_profile(_profile, *_file_cache_statistics); + auto file_cache_delta = io::diff_file_cache_statistics(*_file_cache_statistics, + _reported_file_cache_statistics); + // Profile collection can run more than once. Keep additive fields incremental while + // publishing high-water gauges and peer identities from the latest complete snapshot. + file_cache_delta.remote_only_on_miss_triggered = + _file_cache_statistics->remote_only_on_miss_triggered; + file_cache_delta.remote_only_on_miss_threshold_bytes = + _file_cache_statistics->remote_only_on_miss_threshold_bytes; + file_cache_delta.peer_hosts = _file_cache_statistics->peer_hosts; + _report_file_cache_profile(_profile, file_cache_delta); _state->get_query_ctx()->resource_ctx()->io_context()->update_bytes_write_into_cache( - _file_cache_statistics->bytes_write_into_cache); + file_cache_delta.bytes_write_into_cache); + _reported_file_cache_statistics = *_file_cache_statistics; } if (_file_reader_stats != nullptr) { COUNTER_SET(_file_read_bytes_counter, cast_set(_file_reader_stats->read_bytes)); COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); + const auto read_time = cast_set(_file_reader_stats->read_time_ns); + DORIS_CHECK(read_time >= _reported_io_read_time); + // Some transports (for example Arrow Flight) record directly into IO, while filesystem + // reads arrive through FileReaderStats. Add only the new traced delta so both paths remain + // visible without double counting repeated profile publication. + COUNTER_UPDATE(_io_timer, read_time - _reported_io_read_time); + _reported_io_read_time = read_time; } // Query profiles can be collected before Scanner::close() runs. Publish condition-cache // counters here as well, using deltas so this method and close() cannot double count. @@ -1000,7 +1218,8 @@ void FileScannerV2::_collect_profile_before_close() { void FileScannerV2::_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics) { - io::FileCacheProfileReporter cache_profile(profile); + file_scan_profile::ensure_hierarchy(profile); + io::FileCacheProfileReporter cache_profile(profile, file_scan_profile::IO); cache_profile.update(&file_cache_statistics); } @@ -1022,9 +1241,8 @@ void FileScannerV2::_report_file_reader_predicate_filtered_rows() { const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0; const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows; if (filtered_delta > 0) { - // File readers can evaluate localized conjuncts before a block reaches Scanner. Count - // those rows as scanner-level unselected rows so load statistics stay identical no matter - // whether a predicate is pushed down or evaluated by Scanner::_filter_output_block(). + // FileReader and TableReader both report their owned predicate rows through the shared IO + // context. Preserve scanner-level load statistics without re-evaluating either predicate. _counter.num_rows_unselected += filtered_delta; _reported_predicate_filtered_rows = filtered_rows; } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 2bddc5d5e69e6c..c68cfb1db8b47c 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -87,6 +87,23 @@ class FileScannerV2 final : public Scanner { static void TEST_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics); static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); + static bool TEST_should_skip_empty(const Status& status, bool stopped); + static bool TEST_should_run_adaptive_batch_size(bool predictor_initialized, + bool current_split_uses_metadata_count) { + return _should_run_adaptive_batch_size(predictor_initialized, + current_split_uses_metadata_count); + } + void TEST_set_scanner_conjuncts(VExprContextSPtrs conjuncts) { + _conjuncts = std::move(conjuncts); + _initialize_scanner_residual_conjuncts(); + } + Status TEST_filter_output_block(Block* block) { return _filter_output_block(block); } + size_t TEST_table_reader_owned_conjunct_count() const { + return _table_reader_owned_conjunct_count; + } + size_t TEST_scanner_residual_conjunct_count() const { + return _scanner_residual_conjuncts.size(); + } #endif FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -104,6 +121,9 @@ class FileScannerV2 final : public Scanner { protected: Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) override; + Status _filter_output_block(Block* block) override; + size_t _last_block_rows_read(const Block& block) const override; + size_t _last_block_bytes_read(const Block& block) const override; void _collect_profile_before_close() override; bool _should_update_load_counters() const override; @@ -121,6 +141,7 @@ class FileScannerV2 final : public Scanner { Status _prepare_table_reader_split(const TFileRangeDesc& range, std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); + static bool _should_skip_empty(const Status& status, bool stopped); bool _should_enable_file_meta_cache() const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; @@ -132,12 +153,20 @@ class FileScannerV2 final : public Scanner { Status _build_default_expr(const TFileScanSlotInfo& slot_info, VExprContextSPtr* ctx) const; static format::ColumnDefinition _build_table_column(const SlotDescriptor* slot_desc); Status _build_table_conjuncts(VExprContextSPtrs* conjuncts) const; + Status _build_table_conjuncts(const VExprContextSPtrs& source, + VExprContextSPtrs* conjuncts) const; + Status _sync_table_reader_conjuncts(); + static size_t _safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts); + void _initialize_scanner_residual_conjuncts(); + void _refresh_scanner_residual_profile(); static Status _to_file_format(TFileFormatType::type format_type, format::FileFormat* file_format); void _reset_adaptive_batch_size_state(); void _init_adaptive_batch_size_state(TFileFormatType::type format_type); bool _should_enable_adaptive_batch_size(TFileFormatType::type format_type) const; bool _should_run_adaptive_batch_size() const; + static bool _should_run_adaptive_batch_size(bool predictor_initialized, + bool current_split_uses_metadata_count); size_t _predict_reader_batch_rows(); void _update_adaptive_batch_size(const Block& block); static RealtimeCounterDeltas _collect_realtime_counter_deltas( @@ -165,6 +194,10 @@ class FileScannerV2 final : public Scanner { std::string _current_range_path; std::unique_ptr _table_reader; + size_t _table_reader_owned_conjunct_count = 0; + // Scanner owns one persistent context vector for the first unsafe conjunct and every later + // conjunct. Hybrid child readers may be recreated or switched, but this state must not be. + VExprContextSPtrs _scanner_residual_conjuncts; std::vector _projected_columns; // File formats without embedded schema, such as CSV, still need the FE slot descriptors in // file-column order. This mirrors old FileScanner::_file_slot_descs and is passed only to @@ -176,11 +209,20 @@ class FileScannerV2 final : public Scanner { std::unordered_map _partition_slot_descs; std::unique_ptr _file_cache_statistics; + io::FileCacheStatistics _reported_file_cache_statistics; std::unique_ptr _file_reader_stats; std::shared_ptr _io_ctx; ShardedKVCache* _kv_cache = nullptr; + RuntimeProfile::Counter* _scanner_total_timer = nullptr; + RuntimeProfile::Counter* _init_timer = nullptr; + RuntimeProfile::Counter* _open_timer = nullptr; RuntimeProfile::Counter* _get_block_timer = nullptr; + RuntimeProfile::Counter* _empty_file_counter = nullptr; + RuntimeProfile::Counter* _prepare_split_timer = nullptr; + RuntimeProfile::Counter* _get_next_range_timer = nullptr; + RuntimeProfile::Counter* _close_timer = nullptr; + RuntimeProfile::Counter* _io_timer = nullptr; RuntimeProfile::Counter* _not_found_file_counter = nullptr; RuntimeProfile::Counter* _file_counter = nullptr; RuntimeProfile::Counter* _file_read_bytes_counter = nullptr; @@ -189,6 +231,9 @@ class FileScannerV2 final : public Scanner { RuntimeProfile::Counter* _adaptive_batch_predicted_rows_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_actual_bytes_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_probe_count_counter = nullptr; + RuntimeProfile::Counter* _scanner_residual_filter_timer = nullptr; + RuntimeProfile::Counter* _scanner_residual_rows_filtered_counter = nullptr; + RuntimeProfile* _scanner_profile = nullptr; std::unique_ptr _block_size_predictor; int64_t _reported_predicate_filtered_rows = 0; int64_t _reported_condition_cache_hit_count = 0; @@ -197,6 +242,8 @@ class FileScannerV2 final : public Scanner { int64_t _last_read_rows = 0; int64_t _last_bytes_read_from_local = 0; int64_t _last_bytes_read_from_remote = 0; + int64_t _reported_io_read_time = 0; + int _table_reader_applied_rf_num = 0; }; } // namespace doris diff --git a/be/src/exec/scan/scanner.cpp b/be/src/exec/scan/scanner.cpp index 8788331c24546d..138b3910d4f2fb 100644 --- a/be/src/exec/scan/scanner.cpp +++ b/be/src/exec/scan/scanner.cpp @@ -19,6 +19,8 @@ #include +#include + #include "common/config.h" #include "common/status.h" #include "core/block/column_with_type_and_name.h" @@ -160,8 +162,11 @@ Status Scanner::get_block(RuntimeState* state, Block* block, bool* eof) { DCHECK(block->rows() == 0); break; } - _num_rows_read += block->rows(); - _num_byte_read += block->allocated_bytes(); + // Some scanners apply owned predicates before returning the block. Account the + // materialized input, not only survivors, so the per-turn progress bound remains + // effective for highly selective predicates. + _num_rows_read += _last_block_rows_read(*block); + _num_byte_read += _last_block_bytes_read(*block); } // 2. Filter the output block finally. @@ -256,7 +261,9 @@ Status Scanner::try_append_late_arrival_runtime_filter() { } DCHECK(_applied_rf_num < _total_rf_num); int arrived_rf_num = 0; - RETURN_IF_ERROR(_local_state->update_late_arrival_runtime_filter(_state, arrived_rf_num)); + VExprContextSPtrs arrived_conjuncts; + RETURN_IF_ERROR(_local_state->update_late_arrival_runtime_filter( + _state, _applied_rf_num, arrived_rf_num, arrived_conjuncts)); if (arrived_rf_num == _applied_rf_num) { // No newly arrived runtime filters, just return; @@ -266,10 +273,47 @@ Status Scanner::try_append_late_arrival_runtime_filter() { // avoid conjunct destroy in used by storage layer _conjuncts.clear(); RETURN_IF_ERROR(_local_state->clone_conjunct_ctxs(_conjuncts)); + _late_arrival_rf_conjuncts.insert(_late_arrival_rf_conjuncts.end(), + std::make_move_iterator(arrived_conjuncts.begin()), + std::make_move_iterator(arrived_conjuncts.end())); _applied_rf_num = arrived_rf_num; return Status::OK(); } +uint64_t Scanner::_current_condition_cache_digest() const { + DORIS_CHECK(_state != nullptr); + DORIS_CHECK(_local_state != nullptr); + if (_local_state->get_condition_cache_digest() == 0) { + return 0; + } + + // ScanLocalState computed its digest after collecting the RFs that were ready during open(). A + // scanner may later clone more RF conjuncts between file splits, so rebuild from its current + // snapshot instead of reusing that stale value. For example, split 0 may use P, while split 1 + // starts after an IN RF with payload {7, 9} arrives and must use digest(P AND RF{7, 9}). A + // different payload {8, 10} consequently receives a different key. get_digest() returning zero + // is the correctness fallback for an RF whose complete semantics cannot be represented. + return _build_condition_cache_digest(_state->query_options().condition_cache_digest, + _conjuncts); +} + +uint64_t Scanner::_build_condition_cache_digest(uint64_t seed, const VExprContextSPtrs& conjuncts) { + for (const auto& conjunct : conjuncts) { + seed = conjunct->get_digest(seed); + if (seed == 0) { + return 0; + } + } + return seed; +} + +#ifdef BE_TEST +uint64_t Scanner::TEST_build_condition_cache_digest(uint64_t seed, + const VExprContextSPtrs& conjuncts) { + return _build_condition_cache_digest(seed, conjuncts); +} +#endif + Status Scanner::close(RuntimeState* state) { #ifndef BE_TEST COUNTER_UPDATE(_local_state->_scanner_wait_worker_timer, _scanner_wait_worker_timer); diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index a91395b911efe7..8f9753cf1122a7 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -97,7 +97,19 @@ class Scanner { // eg, for file scanner, return the current file path. virtual std::string get_current_scan_range_name() { return "not implemented"; } +#ifdef BE_TEST + static uint64_t TEST_build_condition_cache_digest(uint64_t seed, + const VExprContextSPtrs& conjuncts); +#endif + protected: + // Rebuild the condition-cache digest from the scanner's current conjunct snapshot. The local + // state's digest is used only as a safety gate: zero means condition cache was disabled during + // scan-node open (for example by TopN or an expression without a reliable digest). + uint64_t _current_condition_cache_digest() const; + static uint64_t _build_condition_cache_digest(uint64_t seed, + const VExprContextSPtrs& conjuncts); + virtual Status _prepare_impl() { _has_prepared = true; return Status::OK(); @@ -138,7 +150,7 @@ class Scanner { bool _try_close(); // Filter the output block finally. - Status _filter_output_block(Block* block); + virtual Status _filter_output_block(Block* block); Status _do_projections(Block* origin_block, Block* output_block); @@ -219,6 +231,11 @@ class Scanner { void update_block_avg_bytes(size_t block_avg_bytes) { _block_avg_bytes = block_avg_bytes; } protected: + virtual size_t _last_block_rows_read(const Block& block) const { return block.rows(); } + virtual size_t _last_block_bytes_read(const Block& block) const { + return block.allocated_bytes(); + } + RuntimeState* _state = nullptr; ScanLocalStateBase* _local_state = nullptr; @@ -246,6 +263,9 @@ class Scanner { // Cloned from _conjuncts of scan node. // It includes predicate in SQL and runtime filters. VExprContextSPtrs _conjuncts; + // Exact append-only RF delta for readers that preserve state across multiple splits. It must + // not be reconstructed by position from the cost-sorted full conjunct snapshot. + VExprContextSPtrs _late_arrival_rf_conjuncts; VExprContextSPtrs _projections; // Used in common subexpression elimination to compute intermediate results. std::vector _intermediate_projections; diff --git a/be/src/exec/sink/viceberg_delete_sink.cpp b/be/src/exec/sink/viceberg_delete_sink.cpp index e2b142db8d9f86..ef167ecdbcaea3 100644 --- a/be/src/exec/sink/viceberg_delete_sink.cpp +++ b/be/src/exec/sink/viceberg_delete_sink.cpp @@ -35,6 +35,7 @@ #include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" #include "exprs/vexpr.h" +#include "format/table/deletion_vector.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format/transformer/vfile_format_transformer.h" #include "io/file_factory.h" @@ -106,6 +107,20 @@ Status load_rewritable_delete_rows(RuntimeState* state, RuntimeProfile* profile, } // namespace +Status calculate_iceberg_deletion_vector_content_size(size_t bitmap_size, int64_t* content_size) { + DORIS_CHECK(content_size != nullptr); + constexpr size_t max_bitmap_size = static_cast(MAX_ICEBERG_DELETION_VECTOR_BYTES) - + ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES; + if (bitmap_size > max_bitmap_size) { + return Status::NotSupported( + "Iceberg deletion vector bitmap size exceeds Doris supported limit: {}, " + "maximum bitmap size: {}, content size limit: {}", + bitmap_size, max_bitmap_size, MAX_ICEBERG_DELETION_VECTOR_BYTES); + } + *content_size = static_cast(bitmap_size + ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES); + return Status::OK(); +} + VIcebergDeleteSink::VIcebergDeleteSink(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs, std::shared_ptr dep, @@ -592,7 +607,8 @@ Status VIcebergDeleteSink::_write_deletion_vector_files( blob.partition_spec_id = deletion.partition_spec_id; blob.partition_data_json = deletion.partition_data_json; blob.merged_count = static_cast(merged_rows.cardinality()); - blob.content_size_in_bytes = static_cast(4 + 4 + bitmap_size + 4); + RETURN_IF_ERROR(calculate_iceberg_deletion_vector_content_size( + bitmap_size, &blob.content_size_in_bytes)); blob.blob_data.resize(static_cast(blob.content_size_in_bytes)); merged_rows.write(blob.blob_data.data() + 8); @@ -604,7 +620,7 @@ Status VIcebergDeleteSink::_write_deletion_vector_files( uint32_t crc = static_cast( ::crc32(0, reinterpret_cast(blob.blob_data.data() + 4), - 4 + (uInt)bitmap_size)); + static_cast(4 + bitmap_size))); BigEndian::Store32(blob.blob_data.data() + 8 + bitmap_size, crc); blobs.emplace_back(std::move(blob)); } diff --git a/be/src/exec/sink/viceberg_delete_sink.h b/be/src/exec/sink/viceberg_delete_sink.h index 22ae98cc288100..647ee92f525733 100644 --- a/be/src/exec/sink/viceberg_delete_sink.h +++ b/be/src/exec/sink/viceberg_delete_sink.h @@ -20,6 +20,8 @@ #include #include +#include +#include #include #include #include @@ -41,6 +43,8 @@ class Dependency; class VIcebergDeleteFileWriter; +Status calculate_iceberg_deletion_vector_content_size(size_t bitmap_size, int64_t* content_size); + struct IcebergFileDeletion { IcebergFileDeletion() = default; IcebergFileDeletion(int32_t spec_id, std::string data_json) diff --git a/be/src/exec/sink/viceberg_merge_sink.cpp b/be/src/exec/sink/viceberg_merge_sink.cpp index 7d6c84cf068dbc..e9f4d6bf5c655a 100644 --- a/be/src/exec/sink/viceberg_merge_sink.cpp +++ b/be/src/exec/sink/viceberg_merge_sink.cpp @@ -254,6 +254,9 @@ Status VIcebergMergeSink::_build_inner_sinks() { if (merge_sink.__isset.broker_addresses) { table_sink.__set_broker_addresses(merge_sink.broker_addresses); } + if (merge_sink.__isset.collect_column_stats) { + table_sink.__set_collect_column_stats(merge_sink.collect_column_stats); + } _table_sink.__set_type(TDataSinkType::ICEBERG_TABLE_SINK); _table_sink.__set_iceberg_table_sink(table_sink); diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp index 5f28c41c24d61b..5958165ad0395c 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -47,7 +47,11 @@ VIcebergPartitionWriter::VIcebergPartitionWriter( _file_name_index(file_name_index), _file_format_type(file_format_type), _compress_type(compress_type), - _hadoop_conf(hadoop_conf) {} + _hadoop_conf(hadoop_conf) { + if (t_sink.iceberg_table_sink.__isset.collect_column_stats) { + _collect_column_stats = t_sink.iceberg_table_sink.collect_column_stats; + } +} Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profile, const RowDescriptor* row_desc) { @@ -156,6 +160,10 @@ Status VIcebergPartitionWriter::_build_iceberg_commit_data(TIcebergCommitData* c commit_data->__set_file_size(_file_format_transformer->written_len()); commit_data->__set_file_content(TFileContent::DATA); commit_data->__set_partition_values(_partition_values); + // ORC collection reopens the file, so honor the FE policy before any footer work. + if (!_collect_column_stats) { + return Status::OK(); + } if (_file_format_type == TFileFormatType::FORMAT_PARQUET) { TIcebergColumnStats column_stats; RETURN_IF_ERROR(static_cast(_file_format_transformer.get()) diff --git a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h index 97b4dd3efdfac0..b0839a82ed3796 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_partition_writer.h @@ -67,6 +67,8 @@ class VIcebergPartitionWriter : public IPartitionWriterBase { inline size_t written_len() const override { return _file_format_transformer->written_len(); } private: + friend class VIcebergPartitionWriterTest; + std::string _get_target_file_name(); Status _build_iceberg_commit_data(TIcebergCommitData* commit_data); @@ -91,6 +93,7 @@ class VIcebergPartitionWriter : public IPartitionWriterBase { TFileFormatType::type _file_format_type; TFileCompressType::type _compress_type; const std::map& _hadoop_conf; + bool _collect_column_stats = true; std::shared_ptr _fs = nullptr; diff --git a/be/src/exec/sink/writer/vtablet_writer.cpp b/be/src/exec/sink/writer/vtablet_writer.cpp index 9f86cd707d295a..5bd43f88726625 100644 --- a/be/src/exec/sink/writer/vtablet_writer.cpp +++ b/be/src/exec/sink/writer/vtablet_writer.cpp @@ -1266,14 +1266,12 @@ void VNodeChannel::_add_block_success_callback(const PTabletWriterAddBlockResult if (!st.ok()) { _cancel_with_msg(st.to_string()); } else if (ctx._is_last_rpc) { - bool skip_tablet_info = false; - DBUG_EXECUTE_IF("VNodeChannel.add_block_success_callback.incomplete_commit_info", - { skip_tablet_info = true; }); for (const auto& tablet : result.tablet_vec()) { DBUG_EXECUTE_IF("VNodeChannel.add_block_success_callback.incomplete_commit_info", { - if (skip_tablet_info) { - LOG(INFO) << "skip tablet info: " << tablet.tablet_id(); - skip_tablet_info = false; + auto target_tablet_id = dp->param("tablet_id", -1); + if (tablet.tablet_id() == target_tablet_id) { + LOG(INFO) << "skip tablet info: " << tablet.tablet_id() + << ", backend_id: " << _node_id; continue; } }); diff --git a/be/src/exprs/aggregate/aggregate_function_avg.h b/be/src/exprs/aggregate/aggregate_function_avg.h index 876bf1ede7b8d7..795dce34ec358f 100644 --- a/be/src/exprs/aggregate/aggregate_function_avg.h +++ b/be/src/exprs/aggregate/aggregate_function_avg.h @@ -29,6 +29,7 @@ #include #include +#include "common/compiler_util.h" #include "core/assert_cast.h" #include "core/column/column.h" #include "core/column/column_fixed_length_object.h" @@ -167,9 +168,7 @@ class AggregateFunctionAvg final template NO_SANITIZE_UNDEFINED void update_value(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num) const { -#ifdef __clang__ -#pragma clang fp reassociate(on) -#endif + ALLOW_FP_REASSOCIATION const auto& column = assert_cast(*columns[0]); if constexpr (is_add) { @@ -262,6 +261,7 @@ class AggregateFunctionAvg final NO_SANITIZE_UNDEFINED void deserialize_and_merge_from_column_range( AggregateDataPtr __restrict place, const IColumn& column, size_t begin, size_t end, Arena&) const override { + ALLOW_FP_REASSOCIATION DCHECK(end <= column.size() && begin <= end) << ", begin:" << begin << ", end:" << end << ", column.size():" << column.size(); auto& col = assert_cast(column); diff --git a/be/src/exprs/aggregate/aggregate_function_avg_weighted.h b/be/src/exprs/aggregate/aggregate_function_avg_weighted.h index 3984b4de41e2f4..7385281d5b6e4d 100644 --- a/be/src/exprs/aggregate/aggregate_function_avg_weighted.h +++ b/be/src/exprs/aggregate/aggregate_function_avg_weighted.h @@ -26,6 +26,7 @@ #include #include +#include "common/compiler_util.h" #include "common/status.h" #include "core/assert_cast.h" #include "core/binary_cast.hpp" @@ -47,9 +48,7 @@ template struct AggregateFunctionAvgWeightedData { using DataType = typename PrimitiveTypeTraits::CppType; void add(const DataType& data_val, double weight_val) { -#ifdef __clang__ -#pragma clang fp reassociate(on) -#endif + ALLOW_FP_REASSOCIATION data_sum = data_sum + (double(data_val) * weight_val); weight_sum = weight_sum + weight_val; } @@ -65,9 +64,7 @@ struct AggregateFunctionAvgWeightedData { } void merge(const AggregateFunctionAvgWeightedData& rhs) { -#ifdef __clang__ -#pragma clang fp reassociate(on) -#endif + ALLOW_FP_REASSOCIATION data_sum = data_sum + rhs.data_sum; weight_sum = weight_sum + rhs.weight_sum; } diff --git a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp index 4b6917f396298d..cf27191ebe74b9 100644 --- a/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp +++ b/be/src/exprs/aggregate/aggregate_function_python_udaf.cpp @@ -33,6 +33,7 @@ #include "core/data_type/define_primitive_type.h" #include "format/arrow/arrow_block_convertor.h" #include "format/arrow/arrow_row_batch.h" +#include "runtime/exec_env.h" #include "runtime/user_function_cache.h" #include "udf/python/python_env.h" #include "udf/python/python_server.h" @@ -66,8 +67,9 @@ Status AggregatePythonUDAFData::add(int64_t place_id, const IColumn** columns, std::shared_ptr batch; // Zero-copy: convert only the specified range - RETURN_IF_ERROR(convert_to_arrow_batch(input_block, schema, arrow::default_memory_pool(), - &batch, timezone_obj, row_num_start, row_num_end)); + RETURN_IF_ERROR(convert_to_arrow_batch(input_block, schema, + ExecEnv::GetInstance()->arrow_memory_pool(), &batch, + timezone_obj, row_num_start, row_num_end)); // Send the batch (already sliced in convert_to_arrow_batch) // Single place mode: no places column needed RETURN_IF_ERROR(client->accumulate(place_id, true, *batch, 0, batch->num_rows())); @@ -111,8 +113,9 @@ Status AggregatePythonUDAFData::add_batch(AggregateDataPtr* places, size_t place std::shared_ptr batch; // Zero-copy: convert only the [start, end) range // This slice includes the places column automatically - RETURN_IF_ERROR(convert_to_arrow_batch(input_block, schema, arrow::default_memory_pool(), - &batch, timezone_obj, start, end)); + RETURN_IF_ERROR(convert_to_arrow_batch(input_block, schema, + ExecEnv::GetInstance()->arrow_memory_pool(), &batch, + timezone_obj, start, end)); // Send entire batch (already contains places column) to Python // place_id=0 is ignored when is_single_place=false RETURN_IF_ERROR(client->accumulate(0, false, *batch, 0, slice_rows)); diff --git a/be/src/exprs/aggregate/aggregate_function_sum.h b/be/src/exprs/aggregate/aggregate_function_sum.h index ca7c1aaf557dbd..8eddb893ed9849 100644 --- a/be/src/exprs/aggregate/aggregate_function_sum.h +++ b/be/src/exprs/aggregate/aggregate_function_sum.h @@ -25,6 +25,7 @@ #include #include +#include "common/compiler_util.h" #include "core/assert_cast.h" #include "core/column/column.h" #include "core/data_type/data_type.h" @@ -51,9 +52,7 @@ struct AggregateFunctionSumData { typename PrimitiveTypeTraits::CppType sum {}; NO_SANITIZE_UNDEFINED void add(typename PrimitiveTypeTraits::CppType value) { -#ifdef __clang__ -#pragma clang fp reassociate(on) -#endif + ALLOW_FP_REASSOCIATION sum += value; } diff --git a/be/src/exprs/block_bloom_filter_impl.cc b/be/src/exprs/block_bloom_filter_impl.cc index d0d8d4245e949c..84098e716caadc 100644 --- a/be/src/exprs/block_bloom_filter_impl.cc +++ b/be/src/exprs/block_bloom_filter_impl.cc @@ -27,10 +27,12 @@ #include // IWYU pragma: keep #include #include -#include #include +#include "common/exception.h" #include "common/status.h" +#include "core/allocator.h" +#include "core/allocator_fwd.h" #include "exprs/block_bloom_filter.hpp" // IWYU pragma: no_include #include "util/sse_util.hpp" @@ -56,24 +58,25 @@ BlockBloomFilter::~BlockBloomFilter() { Status BlockBloomFilter::init_internal(const int log_space_bytes, uint32_t hash_seed) { // Since log_space_bytes is in bytes, we need to convert it to the number of tiny // Bloom filters we will use. - _log_num_buckets = std::max(1, log_space_bytes - kLogBucketByteSize); + const int log_num_buckets = std::max(1, log_space_bytes - kLogBucketByteSize); // Since we use 32 bits in the arguments of Insert() and Find(), _log_num_buckets // must be limited. - if (_log_num_buckets > 32) { + if (log_num_buckets > 32) { return Status::InvalidArgument("Bloom filter too large. log_space_bytes: {}", log_space_bytes); } + + close(); // Ensure that any previously allocated memory for directory_ is released. + DCHECK(_directory == nullptr); + + _log_num_buckets = log_num_buckets; // Don't use _log_num_buckets if it will lead to undefined behavior by a shift // that is too large. _directory_mask = (1 << _log_num_buckets) - 1; const size_t alloc_size = directory_size(); - close(); // Ensure that any previously allocated memory for directory_ is released. - DCHECK(_directory == nullptr); - int rc = posix_memalign((void**)&_directory, 32, alloc_size); - if (rc != 0) { - return Status::InternalError("block_bloom_filter alloc fail"); - } + RETURN_IF_CATCH_EXCEPTION(_directory = reinterpret_cast( + Allocator {}.alloc(alloc_size, kBucketByteSize))); _hash_seed = hash_seed; return Status::OK(); @@ -113,7 +116,7 @@ Status BlockBloomFilter::init_from_directory(int log_space_bytes, void BlockBloomFilter::close() { if (_directory != nullptr) { - free(_directory); + Allocator {}.free(_directory, directory_size()); _directory = nullptr; } } diff --git a/be/src/exprs/function/array/function_array_difference.h b/be/src/exprs/function/array/function_array_difference.h index 065be5a35be3de..cb29ef42737cfb 100644 --- a/be/src/exprs/function/array/function_array_difference.h +++ b/be/src/exprs/function/array/function_array_difference.h @@ -130,14 +130,11 @@ class FunctionArrayDifference : public IFunction { template NO_SANITIZE_UNDEFINED static void impl(const Element* __restrict src, Result* __restrict dst, size_t begin, size_t end) { - size_t curr_pos = begin; - if (curr_pos < end) { - Element prev_element = src[curr_pos]; - dst[curr_pos] = {}; - curr_pos++; - Element curr_element = src[curr_pos]; - for (; curr_pos < end; ++curr_pos) { - curr_element = src[curr_pos]; + if (begin < end) { + Element prev_element = src[begin]; + dst[begin] = {}; + for (size_t curr_pos = begin + 1; curr_pos < end; ++curr_pos) { + Element curr_element = src[curr_pos]; dst[curr_pos] = static_cast(curr_element) - static_cast(prev_element); prev_element = curr_element; diff --git a/be/src/exprs/function/cast/cast_to_array.h b/be/src/exprs/function/cast/cast_to_array.h index c1d96c25f35e55..d4eb688653044c 100644 --- a/be/src/exprs/function/cast/cast_to_array.h +++ b/be/src/exprs/function/cast/cast_to_array.h @@ -16,11 +16,19 @@ // under the License. #include "core/column/column_array.h" -#include "core/column/column_nullable.h" #include "core/data_type/data_type_array.h" #include "exprs/function/cast/cast_base.h" namespace doris::CastWrapper { +inline bool has_null_literal_leaf(const DataTypePtr& type) { + const auto type_without_nullable = remove_nullable(type); + if (const auto* array_type = + check_and_get_data_type(type_without_nullable.get())) { + return has_null_literal_leaf(array_type->get_nested_type()); + } + return type_without_nullable->is_null_literal(); +} + WrapperType create_array_wrapper(FunctionContext* context, const DataTypePtr& from_type_untyped, const DataTypeArray& to_type) { /// Conversion from String through parsing. @@ -42,11 +50,8 @@ WrapperType create_array_wrapper(FunctionContext* context, const DataTypePtr& fr DataTypePtr from_nested_type = from_type->get_nested_type(); - /// In query SELECT CAST([] AS Array(Array(String))) from type is Array(Nothing) - bool from_empty_array = from_nested_type->get_primitive_type() == INVALID_TYPE; - if (from_type->get_number_of_dimensions() != to_type.get_number_of_dimensions() && - !from_empty_array) { + !has_null_literal_leaf(from_nested_type)) { return CastWrapper::create_unsupport_wrapper( "CAST AS Array can only be performed between same-dimensional array types"); } @@ -90,4 +95,4 @@ WrapperType create_array_wrapper(FunctionContext* context, const DataTypePtr& fr return Status::OK(); }; } -} // namespace doris::CastWrapper \ No newline at end of file +} // namespace doris::CastWrapper diff --git a/be/src/exprs/function/function_bitmap.cpp b/be/src/exprs/function/function_bitmap.cpp index 35341f297640b0..6213066e0e1575 100644 --- a/be/src/exprs/function/function_bitmap.cpp +++ b/be/src/exprs/function/function_bitmap.cpp @@ -323,10 +323,15 @@ struct BitmapFromArray { const auto& nested_column_data = static_cast(nested_column).get_data(); auto size = offset_column_data.size(); res.reserve(size); - std::vector bits; + // Preserve the nested column's native integer type here. + // For Array/Array-like inputs can reach the 32-bit `add_many` fast path + // instead of widening every element to uint64_t first. + using ValueType = typename ColumnType::value_type; + std::vector bits; for (size_t i = 0; i < size; ++i) { auto curr_offset = offset_column_data[i]; auto prev_offset = offset_column_data[i - 1]; + bits.reserve(curr_offset - prev_offset); for (auto j = prev_offset; j < curr_offset; ++j) { auto data = nested_column_data[j]; // invaild value @@ -904,25 +909,19 @@ struct BitmapHasAny { static void vector_vector(const TData& lvec, const TData& rvec, ResTData& res) { size_t size = lvec.size(); for (size_t i = 0; i < size; ++i) { - auto bitmap = lvec[i]; - bitmap &= rvec[i]; - res[i] = bitmap.cardinality() != 0; + res[i] = lvec[i].intersects(rvec[i]); } } static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData& res) { size_t size = lvec.size(); for (size_t i = 0; i < size; ++i) { - auto bitmap = lvec[i]; - bitmap &= rval; - res[i] = bitmap.cardinality() != 0; + res[i] = lvec[i].intersects(rval); } } static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData& res) { size_t size = rvec.size(); for (size_t i = 0; i < size; ++i) { - auto bitmap = lval; - bitmap &= rvec[i]; - res[i] = bitmap.cardinality() != 0; + res[i] = lval.intersects(rvec[i]); } } }; @@ -942,28 +941,19 @@ struct BitmapHasAll { static void vector_vector(const TData& lvec, const TData& rvec, ResTData& res) { size_t size = lvec.size(); for (size_t i = 0; i < size; ++i) { - uint64_t lhs_cardinality = lvec[i].cardinality(); - auto bitmap = lvec[i]; - bitmap |= rvec[i]; - res[i] = bitmap.cardinality() == lhs_cardinality; + res[i] = lvec[i].contains_all(rvec[i]); } } static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData& res) { size_t size = lvec.size(); for (size_t i = 0; i < size; ++i) { - uint64_t lhs_cardinality = lvec[i].cardinality(); - auto bitmap = lvec[i]; - bitmap |= rval; - res[i] = bitmap.cardinality() == lhs_cardinality; + res[i] = lvec[i].contains_all(rval); } } static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData& res) { size_t size = rvec.size(); - uint64_t lhs_cardinality = lval.cardinality(); for (size_t i = 0; i < size; ++i) { - auto bitmap = lval; - bitmap |= rvec[i]; - res[i] = bitmap.cardinality() == lhs_cardinality; + res[i] = lval.contains_all(rvec[i]); } } }; diff --git a/be/src/exprs/function/function_bitmap_min_or_max.h b/be/src/exprs/function/function_bitmap_min_or_max.h index 2132e6a44d4145..76ebf4616b0022 100644 --- a/be/src/exprs/function/function_bitmap_min_or_max.h +++ b/be/src/exprs/function/function_bitmap_min_or_max.h @@ -68,20 +68,24 @@ class FunctionBitmapSingle : public IFunction { private: void execute_straight(const ColumnBitmap* date_column, ColumnInt64* result_column, NullMap& result_null_map, size_t input_rows_count) const { + const auto& data = date_column->get_data(); + auto& result_data = result_column->get_data(); + result_data.resize(input_rows_count); + for (size_t i = 0; i < input_rows_count; i++) { if (result_null_map[i]) { - result_column->insert_default(); + result_data[i] = 0; continue; } - BitmapValue value = date_column->get_element(i); + const BitmapValue& value = data[i]; if (!value.cardinality()) { result_null_map[i] = true; - result_column->insert_default(); + result_data[i] = 0; continue; } - result_column->insert(Field::create_field(Impl::calculate(value))); + result_data[i] = Impl::calculate(value); } } }; diff --git a/be/src/exprs/function/function_python_udf.cpp b/be/src/exprs/function/function_python_udf.cpp index 54ea74ad456214..ca9fa280f082fb 100644 --- a/be/src/exprs/function/function_python_udf.cpp +++ b/be/src/exprs/function/function_python_udf.cpp @@ -30,6 +30,7 @@ #include "common/status.h" #include "core/block/block.h" #include "format/arrow/arrow_block_convertor.h" +#include "runtime/exec_env.h" #include "runtime/user_function_cache.h" #include "udf/python/python_server.h" #include "udf/python/python_udf_client.h" @@ -144,7 +145,8 @@ Status PythonFunctionCall::execute_impl(FunctionContext* context, Block& block, if (arguments.empty()) { RETURN_IF_ERROR(make_zero_column_arrow_batch(schema, input_rows, &input_batch)); } else { - RETURN_IF_ERROR(convert_to_arrow_batch(input_block, schema, arrow::default_memory_pool(), + RETURN_IF_ERROR(convert_to_arrow_batch(input_block, schema, + ExecEnv::GetInstance()->arrow_memory_pool(), &input_batch, _timezone_obj)); } RETURN_IF_ERROR(client->evaluate(*input_batch, &output_batch)); diff --git a/be/src/exprs/lambda_function/lambda_execution_context.h b/be/src/exprs/lambda_function/lambda_execution_context.h new file mode 100644 index 00000000000000..6068c3e47d1590 --- /dev/null +++ b/be/src/exprs/lambda_function/lambda_execution_context.h @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace doris { + +class LambdaExecutionContext { +public: + struct Binding { + std::string name; + int column_position = -1; + }; + + struct Frame { + bool bind_by_name = true; + bool parent_bindings_visible = true; + std::vector argument_bindings; + }; + + struct ResolveResult { + bool searched_named_scope = false; + bool found = false; + int column_position = -1; + }; + + class FrameGuard { + public: + FrameGuard(LambdaExecutionContext& context, Frame frame) : _context(&context) { + _context->push_frame(std::move(frame)); + } + + FrameGuard(FrameGuard&& other) = delete; + FrameGuard& operator=(FrameGuard&& other) = delete; + FrameGuard(const FrameGuard&) = delete; + FrameGuard& operator=(const FrameGuard&) = delete; + + ~FrameGuard() { release(); } + + private: + void release() { + if (_context != nullptr) { + _context->pop_frame(); + _context = nullptr; + } + } + + LambdaExecutionContext* _context; + }; + + void push_frame(Frame frame) { _frames.push_back(std::move(frame)); } + + void pop_frame() { + DCHECK(!_frames.empty()); + _frames.pop_back(); + } + + ResolveResult resolve_column_position(const std::string& name) const { + ResolveResult result; + for (const auto& frame : std::ranges::reverse_view(_frames)) { + result.searched_named_scope |= frame.bind_by_name; + for (const auto& argument_binding : + std::ranges::reverse_view(frame.argument_bindings)) { + if (argument_binding.name == name) { + result.found = true; + result.column_position = argument_binding.column_position; + return result; + } + } + if (!frame.parent_bindings_visible) { + break; + } + } + return result; + } + + void collect_visible_binding_column_positions(std::set& column_positions) const { + for (const auto& _frame : std::ranges::reverse_view(_frames)) { + for (const auto& binding : _frame.argument_bindings) { + if (binding.column_position >= 0) { + column_positions.insert(binding.column_position); + } + } + if (!_frame.parent_bindings_visible) { + break; + } + } + } + +private: + std::vector _frames; +}; + +} // namespace doris diff --git a/be/src/exprs/lambda_function/lambda_function.h b/be/src/exprs/lambda_function/lambda_function.h index 451b61569e9ca8..31976617a5444f 100644 --- a/be/src/exprs/lambda_function/lambda_function.h +++ b/be/src/exprs/lambda_function/lambda_function.h @@ -31,7 +31,7 @@ class LambdaFunction { virtual std::string get_name() const = 0; - virtual doris::Status prepare(RuntimeState* state) { + virtual doris::Status prepare(RuntimeState* state, const VExprSPtrs& children) { batch_size = state->batch_size(); return Status::OK(); } diff --git a/be/src/exprs/lambda_function/varray_map_function.cpp b/be/src/exprs/lambda_function/varray_map_function.cpp index db82c46500062c..1c22c23b7c8eab 100644 --- a/be/src/exprs/lambda_function/varray_map_function.cpp +++ b/be/src/exprs/lambda_function/varray_map_function.cpp @@ -17,9 +17,11 @@ #include #include +#include #include #include +#include "common/check.h" #include "common/status.h" #include "core/assert_cast.h" #include "core/block/block.h" @@ -28,6 +30,7 @@ #include "core/block/columns_with_type_and_name.h" #include "core/column/column.h" #include "core/column/column_array.h" +#include "core/column/column_nothing.h" #include "core/column/column_nullable.h" #include "core/column/column_vector.h" #include "core/data_type/data_type.h" @@ -36,18 +39,17 @@ #include "core/data_type/data_type_number.h" #include "exec/common/util.hpp" #include "exprs/aggregate/aggregate_function.h" +#include "exprs/lambda_function/lambda_execution_context.h" #include "exprs/lambda_function/lambda_function.h" #include "exprs/lambda_function/lambda_function_factory.h" #include "exprs/vcolumn_ref.h" -#include "exprs/vslot_ref.h" +#include "exprs/vexpr_context.h" +#include "exprs/vlambda_function_expr.h" namespace doris { -class VExprContext; // extend a block with all required parameters struct LambdaArgs { - // the lambda function need the column ids of all the slots - std::vector output_slot_ref_indexs; // which line is extended to the original block int64_t current_row_idx = 0; // when a block is filled, the array may be truncated, recording where it was truncated @@ -76,34 +78,18 @@ class ArrayMapFunction : public LambdaFunction { std::string get_name() const override { return name; } + Status prepare(RuntimeState* state, const VExprSPtrs& children) override { + RETURN_IF_ERROR(LambdaFunction::prepare(state, children)); + DCHECK_GE(children.size(), 2); + + return _prepare_lambda_argument_binding(children[0], children.size() - 1, + _lambda_argument_binding); + } + Status execute(VExprContext* context, const Block* block, const Selector* expr_selector, size_t count, ColumnPtr& result_column, const DataTypePtr& result_type, const VExprSPtrs& children) const override { LambdaArgs args_info; - // collect used slot ref in lambda function body - std::vector& output_slot_ref_indexs = args_info.output_slot_ref_indexs; - _collect_slot_ref_column_id(children[0], output_slot_ref_indexs); - - int gap = 0; - if (!output_slot_ref_indexs.empty()) { - auto max_id = std::ranges::max_element(output_slot_ref_indexs); - gap = *max_id + 1; - _set_column_ref_column_id(children[0], gap); - } - - std::vector names(gap); - DataTypes data_types(gap); - - for (int i = 0; i < gap; ++i) { - if (_contains_column_id(output_slot_ref_indexs, i)) { - names[i] = block->get_by_position(i).name; - data_types[i] = block->get_by_position(i).type; - } else { - // padding some mock data to hold the position, like call block#rows function need - names[i] = "temp"; - data_types[i] = std::make_shared(); - } - } ///* array_map(lambda,arg1,arg2,.....) */// //1. child[1:end]->execute(src_block) @@ -126,6 +112,7 @@ class ArrayMapFunction : public LambdaFunction { ColumnPtr first_array_offsets = nullptr; //2. get the result column from executed expr, and the needed is nested column of array std::vector lambda_datas(arguments.size()); + DataTypes lambda_argument_types(arguments.size()); for (int i = 0; i < arguments.size(); ++i) { const auto& array_column_type_name = arguments[i]; @@ -153,7 +140,6 @@ class ArrayMapFunction : public LambdaFunction { // here is the array column const auto& col_array = assert_cast(*column_array); - const auto& col_type = assert_cast(*type_array); if (i == 0) { nested_array_column_rows = col_array.get_data_ptr()->size(); @@ -180,15 +166,71 @@ class ArrayMapFunction : public LambdaFunction { } } lambda_datas[i] = col_array.get_data_ptr(); - names.push_back("R" + array_column_type_name.name); - data_types.push_back(col_type.get_nested_type()); + const auto& col_type = assert_cast(*type_array); + lambda_argument_types[i] = col_type.get_nested_type(); + } + std::set required_input_column_ids; + children[0]->collect_slot_column_ids(required_input_column_ids); + context->lambda_execution_context().collect_visible_binding_column_positions( + required_input_column_ids); + const int lambda_argument_base = + required_input_column_ids.empty() ? 0 : *required_input_column_ids.rbegin() + 1; + if (!_lambda_argument_binding.bind_by_name) { + RETURN_IF_ERROR( + _set_legacy_lambda_argument_gap(children[0]->get_child(0), lambda_argument_base, + _lambda_argument_binding.argument_size)); + } + std::vector names(lambda_argument_base); + DataTypes data_types(lambda_argument_base); + std::vector materialized_input_columns(lambda_argument_base, false); + names.reserve(lambda_argument_base + arguments.size()); + data_types.reserve(lambda_argument_base + arguments.size()); + for (int column_id : required_input_column_ids) { + if (column_id < 0 || static_cast(column_id) >= block->columns()) { + return Status::InternalError( + "array_map lambda input column id {} is outside input block, block={}", + column_id, block->dump_structure()); + } + materialized_input_columns[column_id] = true; + names[column_id] = block->get_by_position(column_id).name; + data_types[column_id] = block->get_by_position(column_id).type; + } + for (int i = 0; i < lambda_argument_base; ++i) { + if (!materialized_input_columns[i]) { + // Keep sparse input positions stable for SlotRef/parent lambda bindings without + // materializing unrelated wide-table columns into every lambda batch. + names[i] = "temp"; + data_types[i] = std::make_shared(); + } + } + for (int i = 0; i < arguments.size(); ++i) { + const auto& array_column_type_name = arguments[i]; + if (_lambda_argument_binding.bind_by_name && + i < _lambda_argument_binding.names.size()) { + names.push_back(_lambda_argument_binding.names[i]); + } else { + names.push_back("R" + array_column_type_name.name); + } + data_types.push_back(lambda_argument_types[i]); + } + + LambdaExecutionContext::Frame lambda_frame; + lambda_frame.bind_by_name = _lambda_argument_binding.bind_by_name; + lambda_frame.parent_bindings_visible = true; + for (int i = 0; i < _lambda_argument_binding.argument_size; ++i) { + const int column_position = lambda_argument_base + i; + if (_lambda_argument_binding.bind_by_name) { + lambda_frame.argument_bindings.push_back( + {_lambda_argument_binding.names[i], column_position}); + } } + LambdaExecutionContext::FrameGuard lambda_frame_guard(context->lambda_execution_context(), + std::move(lambda_frame)); // if column_array is NULL, we know the array_data_column will not write any data, // so the column is empty. eg : (x) -> concat('|',x + "1"). if still execute the lambda function, will cause the bolck rows are not equal // the x column is empty, but "|" is const literal, size of column is 1, so the block rows is 1, but the x column is empty, will be coredump. - if (std::any_of(lambda_datas.begin(), lambda_datas.end(), - [](const auto& v) { return v->empty(); })) { + if (std::ranges::any_of(lambda_datas, [](const auto& v) { return v->empty(); })) { DataTypePtr nested_type; bool is_nullable = result_type->is_nullable(); if (is_nullable) { @@ -231,33 +273,26 @@ class ArrayMapFunction : public LambdaFunction { if (mem_reuse) { columns[i] = lambda_block.get_by_position(i).column->assert_mutable(); } else { - if (_contains_column_id(output_slot_ref_indexs, i) || i >= gap) { - // TODO: maybe could create const column, so not insert_many_from when extand data - // but now here handle batch_size of array nested data every time, so maybe have different rows - columns[i] = data_types[i]->create_column(); - } else { - columns[i] = data_types[i] - ->create_column_const_with_default_value(0) - ->assert_mutable(); - } + columns[i] = data_types[i]->create_column(); } } // batch_size of array nested data every time inorder to avoid memory overflow - while (columns[gap]->size() < batch_size) { - long max_step = batch_size - columns[gap]->size(); + while (columns[lambda_argument_base]->size() < batch_size) { + long max_step = batch_size - columns[lambda_argument_base]->size(); long current_step = std::min( max_step, (long)(args_info.cur_size - args_info.current_offset_in_array)); size_t pos = args_info.array_start + args_info.current_offset_in_array; for (int i = 0; i < arguments.size() && current_step > 0; ++i) { - columns[gap + i]->insert_range_from(*lambda_datas[i], pos, current_step); + columns[lambda_argument_base + i]->insert_range_from(*lambda_datas[i], pos, + current_step); } args_info.current_offset_in_array += current_step; args_info.current_repeat_times += current_step; if (args_info.current_offset_in_array >= args_info.cur_size) { args_info.current_row_eos = true; } - _extend_data(columns, block, args_info.current_repeat_times, gap, - args_info.current_row_idx, output_slot_ref_indexs); + _repeat_input_columns(columns, block, args_info.current_repeat_times, + materialized_input_columns, args_info.current_row_idx); args_info.current_repeat_times = 0; if (args_info.current_row_eos) { //current row is end of array, move to next row @@ -329,52 +364,104 @@ class ArrayMapFunction : public LambdaFunction { } private: - bool _contains_column_id(const std::vector& output_slot_ref_indexs, int id) const { - const auto it = std::find(output_slot_ref_indexs.begin(), output_slot_ref_indexs.end(), id); - return it != output_slot_ref_indexs.end(); + struct LambdaArgumentBinding { + bool bind_by_name = true; + size_t argument_size = 0; + std::vector names; + }; + + Status _prepare_lambda_argument_binding(const VExprSPtr& expr, size_t expected_argument_size, + LambdaArgumentBinding& argument_binding) const { + DORIS_CHECK_EQ(expr->node_type(), TExprNodeType::LAMBDA_FUNCTION_EXPR); + const auto* lambda_expr = assert_cast(expr.get()); + + argument_binding.argument_size = 0; + argument_binding.names.clear(); + argument_binding.bind_by_name = lambda_expr->has_argument_names(); + + if (!argument_binding.bind_by_name) { + if (_contains_nested_lambda_call(expr->get_child(0))) { + return Status::InternalError( + "Cannot resolve nested lambda argument without lambda metadata"); + } + argument_binding.argument_size = expected_argument_size; + argument_binding.names.resize(expected_argument_size); + return Status::OK(); + } + + argument_binding.names = lambda_expr->argument_names(); + if (argument_binding.names.size() > expected_argument_size) { + return Status::InternalError( + "lambda argument metadata size exceeds parameter size, maximum={}, actual={}", + expected_argument_size, argument_binding.names.size()); + } + argument_binding.argument_size = argument_binding.names.size(); + if (std::ranges::any_of(argument_binding.names, + [](const auto& argument_name) { return argument_name.empty(); })) { + return Status::InternalError("lambda argument metadata contains empty name"); + } + return Status::OK(); } - void _set_column_ref_column_id(VExprSPtr expr, int gap) const { - for (const auto& child : expr->children()) { - if (child->is_column_ref()) { - auto* ref = static_cast(child.get()); - ref->set_gap(gap); - } else { - _set_column_ref_column_id(child, gap); + Status _set_legacy_lambda_argument_gap(const VExprSPtr& expr, int lambda_argument_base, + size_t argument_size) const { + if (expr->is_column_ref()) { + auto* ref = static_cast(expr.get()); + DORIS_CHECK_GE(ref->column_id(), 0); + DORIS_CHECK_LT(static_cast(ref->column_id()), argument_size); + const int argument_index = ref->column_id(); + ref->set_gap(lambda_argument_base + argument_index - ref->column_id()); + } else { + for (const auto& child : expr->children()) { + RETURN_IF_ERROR(_set_legacy_lambda_argument_gap(child, lambda_argument_base, + argument_size)); } } + return Status::OK(); } - void _collect_slot_ref_column_id(VExprSPtr expr, - std::vector& output_slot_ref_indexs) const { - for (const auto& child : expr->children()) { - if (child->is_slot_ref()) { - const auto* ref = static_cast(child.get()); - output_slot_ref_indexs.push_back(ref->column_id()); - } else { - _collect_slot_ref_column_id(child, output_slot_ref_indexs); - } + bool _is_lambda_call_with_lambda_expr(const VExprSPtr& expr) const { + return expr->node_type() == TExprNodeType::LAMBDA_FUNCTION_CALL_EXPR && + !expr->children().empty() && + expr->children()[0]->node_type() == TExprNodeType::LAMBDA_FUNCTION_EXPR; + } + + bool _contains_nested_lambda_call(const VExprSPtr& expr) const { + if (_is_lambda_call_with_lambda_expr(expr)) { + return true; } + return std::ranges::any_of(expr->children(), [this](const auto& child) { + return _contains_nested_lambda_call(child); + }); } - void _extend_data(std::vector& columns, const Block* block, - int current_repeat_times, int size, int64_t current_row_idx, - const std::vector& output_slot_ref_indexs) const { - if (!current_repeat_times || !size) { + void _repeat_input_columns(std::vector& columns, const Block* block, + int repeat_times, + const std::vector& materialized_input_columns, + int64_t row_idx) const { + if (!repeat_times || materialized_input_columns.empty()) { return; } - for (int i = 0; i < size; i++) { - if (_contains_column_id(output_slot_ref_indexs, i)) { - auto src_column = - block->get_by_position(i).column->convert_to_full_column_if_const(); - columns[i]->insert_many_from(*src_column, current_row_idx, current_repeat_times); - } else { - // must be column const - DCHECK(is_column_const(*columns[i])); - columns[i]->resize(columns[i]->size() + current_repeat_times); + for (size_t i = 0; i < materialized_input_columns.size(); i++) { + if (!materialized_input_columns[i]) { + columns[i]->resize(columns[i]->size() + repeat_times); + continue; } + DORIS_CHECK(block != nullptr); + auto src_column = block->get_by_position(i).column->convert_to_full_column_if_const(); + if (check_and_get_column(src_column.get())) { + // A ColumnNothing in the outer block is a placeholder for an unmaterialized + // virtual column. Keep it as a placeholder in the lambda block as well, so + // VirtualSlotRef can still materialize it lazily if the lambda body reads it. + if (!check_and_get_column(columns[i].get())) { + columns[i] = ColumnNothing::create(columns[i]->size()); + } + } + columns[i]->insert_many_from(*src_column, row_idx, repeat_times); } } + + LambdaArgumentBinding _lambda_argument_binding; }; void register_function_array_map(doris::LambdaFunctionFactory& factory) { diff --git a/be/src/exprs/lambda_function/varray_sort_function.cpp b/be/src/exprs/lambda_function/varray_sort_function.cpp index 6d14609400365c..dfd2ed1a8ae58e 100644 --- a/be/src/exprs/lambda_function/varray_sort_function.cpp +++ b/be/src/exprs/lambda_function/varray_sort_function.cpp @@ -17,6 +17,12 @@ #include +#include +#include +#include +#include +#include + #include "common/status.h" #include "core/assert_cast.h" #include "core/block/block.h" @@ -28,14 +34,16 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "exec/common/util.hpp" +#include "exprs/lambda_function/lambda_execution_context.h" #include "exprs/lambda_function/lambda_function.h" #include "exprs/lambda_function/lambda_function_factory.h" +#include "exprs/vcolumn_ref.h" #include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vlambda_function_expr.h" namespace doris { -class VExprContext; - using ConstColumnVariant = std::variantnode_type(), TExprNodeType::LAMBDA_FUNCTION_EXPR); + const auto* lambda_expr = assert_cast(children[0].get()); + const std::vector* argument_names = + lambda_expr->has_argument_names() ? &lambda_expr->argument_names() : nullptr; + RETURN_IF_ERROR(_set_comparator_argument_gap(children[0]->get_child(0), argument_names)); + return Status::OK(); + } + Status execute(VExprContext* context, const Block* block, const Selector* expr_selector, size_t count, ColumnPtr& result_column, const DataTypePtr& result_type, const VExprSPtrs& children) const override { @@ -129,9 +148,9 @@ class ArraySortFunction : public LambdaFunction { * 1 20 1 nullable(int) * 2 1/-1/0 ... tinyint * The size of a column is always 1; we only need to use it to store the specific values ​​in the array for comparison. - */ + */ Block lambda_block; - for (int i = 0; i <= 2; i++) { + for (int i = 0; i < 2; i++) { lambda_block.insert(ColumnWithTypeAndName(nested_nullable_column.clone_empty(), col_type.get_nested_type(), "temp")); } @@ -148,7 +167,14 @@ class ArraySortFunction : public LambdaFunction { temp_nullmap_data[i]->resize(1); }; - int lambda_res_id = 2; + // array_sort's comparator arguments are represented by ColumnRef column ids 0 and 1. + // They are position-based instead of name-based because FE may reuse the first + // argument name for the second comparator ColumnRef and distinguish them only by id. + LambdaExecutionContext::Frame lambda_frame; + lambda_frame.bind_by_name = false; + lambda_frame.parent_bindings_visible = false; + LambdaExecutionContext::FrameGuard lambda_frame_guard(context->lambda_execution_context(), + std::move(lambda_frame)); // 3. sort array by executing lambda function // During the sorting process, the parameter columns of lambda_block are first populated using prepare_lambda_input, @@ -175,12 +201,14 @@ class ArraySortFunction : public LambdaFunction { } }; + const int lambda_result_base = static_cast(lambda_block.columns()); for (int row = 0; row < input_rows; ++row) { auto start = off_data[row - 1]; auto end = off_data[row]; std::sort(&permutation[start], &permutation[end], [&](size_t i, size_t j) { prepare_lambda_input(i, 0); prepare_lambda_input(j, 1); + int lambda_res_id = lambda_result_base; auto status = children[0]->execute(context, &lambda_block, &lambda_res_id); if (!status.ok()) [[unlikely]] { @@ -197,6 +225,7 @@ class ArraySortFunction : public LambdaFunction { // only -1, 0, 1 long cmp = assert_cast(full_res_col.get()) ->get_data()[0]; + lambda_block.erase_tail(lambda_result_base); return cmp < 0; }); @@ -221,6 +250,146 @@ class ArraySortFunction : public LambdaFunction { return Status::OK(); } +private: + Status _set_comparator_argument_gap(const VExprSPtr& expr, + const std::vector* argument_names) const { + if (expr->is_column_ref()) { + auto* ref = static_cast(expr.get()); + RETURN_IF_ERROR(_validate_comparator_argument_ref(*ref, argument_names)); + ref->set_gap(0); + return Status::OK(); + } + + if (expr->is_slot_ref() || expr->is_virtual_slot_ref()) { + return Status::NotSupported( + "array_sort comparator only supports its own lambda arguments, but found " + "captured slot ref '{}'", + expr->expr_name()); + } + + if (_is_lambda_call_with_lambda_expr(expr)) { + // array_sort comparator arguments live in a position-based, comparator-local frame + // that is invisible to nested lambda frames. Reject unsupported nested captures during + // prepare, otherwise execution would later fail with an internal missing-column error. + // For example, array_sort((x, y) -> array_map(z -> z + x, nested_arr), arr) is + // rejected because the inner array_map lambda captures the comparator-local x; while + // array_sort((x, y) -> array_map(x -> x + 1, nested_arr), arr) is still valid because + // the inner x is array_map's own argument and shadows the comparator argument. + RETURN_IF_ERROR(_reject_nested_lambda_capture_of_comparator_argument( + assert_cast(expr->children()[0].get()), + argument_names)); + for (int i = 1; i < expr->children().size(); ++i) { + RETURN_IF_ERROR(_set_comparator_argument_gap(expr->children()[i], argument_names)); + } + return Status::OK(); + } + + for (const auto& child : expr->children()) { + RETURN_IF_ERROR(_set_comparator_argument_gap(child, argument_names)); + } + return Status::OK(); + } + + Status _reject_nested_lambda_capture_of_comparator_argument( + const VLambdaFunctionExpr* lambda_expr, + const std::vector* comparator_argument_names) const { + if (!lambda_expr->has_argument_names()) { + return Status::InternalError( + "Cannot validate nested lambda capture in array_sort comparator without lambda " + "metadata"); + } + return _reject_nested_lambda_capture_of_comparator_argument(lambda_expr->get_child(0), + comparator_argument_names, + lambda_expr->argument_names()); + } + + Status _reject_nested_lambda_capture_of_comparator_argument( + const VExprSPtr& expr, const std::vector* comparator_argument_names, + const std::vector& in_scope_lambda_argument_names) const { + // Names in in_scope_lambda_argument_names are declared by the nested lambda scopes that + // enclose expr. They can legally shadow array_sort comparator argument names, so a + // ColumnRef matching one of these names should be treated as a local nested-lambda + // argument instead of an unsupported capture from the array_sort comparator. + if (expr->is_column_ref()) { + if (std::ranges::find(in_scope_lambda_argument_names, expr->expr_name()) != + in_scope_lambda_argument_names.end()) { + return Status::OK(); + } + if (comparator_argument_names != nullptr && + std::ranges::find(*comparator_argument_names, expr->expr_name()) != + comparator_argument_names->end()) { + return Status::NotSupported( + "array_sort comparator does not support nested lambda capturing comparator " + "argument '{}'", + expr->expr_name()); + } + return Status::NotSupported( + "array_sort comparator only supports nested lambda arguments inside nested " + "lambda bodies, but found captured column ref '{}'", + expr->expr_name()); + } + + if (expr->is_slot_ref() || expr->is_virtual_slot_ref()) { + return Status::NotSupported( + "array_sort comparator only supports nested lambda arguments inside nested " + "lambda bodies, but found captured slot ref '{}'", + expr->expr_name()); + } + + if (_is_lambda_call_with_lambda_expr(expr)) { + const auto* lambda_expr = + assert_cast(expr->children()[0].get()); + if (!lambda_expr->has_argument_names()) { + return Status::InternalError( + "Cannot validate nested lambda capture in array_sort comparator without " + "lambda metadata"); + } + auto nested_in_scope_lambda_argument_names = in_scope_lambda_argument_names; + nested_in_scope_lambda_argument_names.insert( + nested_in_scope_lambda_argument_names.end(), + lambda_expr->argument_names().begin(), lambda_expr->argument_names().end()); + RETURN_IF_ERROR(_reject_nested_lambda_capture_of_comparator_argument( + lambda_expr->get_child(0), comparator_argument_names, + nested_in_scope_lambda_argument_names)); + for (int i = 1; i < expr->children().size(); ++i) { + RETURN_IF_ERROR(_reject_nested_lambda_capture_of_comparator_argument( + expr->children()[i], comparator_argument_names, + in_scope_lambda_argument_names)); + } + return Status::OK(); + } + + for (const auto& child : expr->children()) { + RETURN_IF_ERROR(_reject_nested_lambda_capture_of_comparator_argument( + child, comparator_argument_names, in_scope_lambda_argument_names)); + } + return Status::OK(); + } + + Status _validate_comparator_argument_ref(const VColumnRef& ref, + const std::vector* argument_names) const { + if (ref.column_id() < 0 || ref.column_id() >= 2) { + return Status::NotSupported( + "array_sort comparator only supports its own lambda arguments, but found " + "column ref '{}' with invalid column id {}", + ref.expr_name(), ref.column_id()); + } + if (argument_names != nullptr && + std::ranges::find(*argument_names, ref.expr_name()) == argument_names->end()) { + return Status::NotSupported( + "array_sort comparator only supports its own lambda arguments, but found " + "captured column ref '{}'", + ref.expr_name()); + } + return Status::OK(); + } + + bool _is_lambda_call_with_lambda_expr(const VExprSPtr& expr) const { + return expr->node_type() == TExprNodeType::LAMBDA_FUNCTION_CALL_EXPR && + !expr->children().empty() && + expr->children()[0]->node_type() == TExprNodeType::LAMBDA_FUNCTION_EXPR; + } + #define DISPATCH_PRIMITIVE_TYPE(TYPE, COLUMN_CLASS) \ case TYPE: \ column_variant = &assert_cast(column); \ diff --git a/be/src/exprs/table_function/python_udtf_function.cpp b/be/src/exprs/table_function/python_udtf_function.cpp index eaaf4d5227a600..fbbea4bfc81f6c 100644 --- a/be/src/exprs/table_function/python_udtf_function.cpp +++ b/be/src/exprs/table_function/python_udtf_function.cpp @@ -38,6 +38,7 @@ #include "format/arrow/arrow_block_convertor.h" #include "format/arrow/arrow_row_batch.h" #include "format/arrow/arrow_utils.h" +#include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "runtime/user_function_cache.h" #include "udf/python/python_env.h" @@ -141,8 +142,8 @@ Status PythonUDTFFunction::process_init(Block* block, RuntimeState* state) { RETURN_IF_ERROR(make_zero_column_arrow_batch(input_schema, input_rows, &input_batch)); } else { RETURN_IF_ERROR(convert_to_arrow_batch(input_block, input_schema, - arrow::default_memory_pool(), &input_batch, - _timezone_obj)); + ExecEnv::GetInstance()->arrow_memory_pool(), + &input_batch, _timezone_obj)); } // Step 3: Call Python UDTF to evaluate all rows at once (similar to Java UDTF's JNI call) diff --git a/be/src/exprs/vcolumn_ref.h b/be/src/exprs/vcolumn_ref.h index 33ade77defaaba..253c50f06270e4 100644 --- a/be/src/exprs/vcolumn_ref.h +++ b/be/src/exprs/vcolumn_ref.h @@ -18,8 +18,11 @@ #pragma once #include +#include "common/exception.h" #include "exprs/function/function.h" +#include "exprs/lambda_function/lambda_execution_context.h" #include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" @@ -59,14 +62,27 @@ class VColumnRef final : public VExpr { Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const override { DCHECK(_open_finished || block == nullptr); - auto origin_column = block->get_by_position(_column_id + _gap).column; + const int column_position = _get_column_position(context, block); + if (column_position < 0 || column_position >= block->columns()) { + return Status::InternalError( + "input block not contain column ref {}, column_id={}, gap={}, block={}", + _column_name, _column_id, _gap.load(), block->dump_structure()); + } + auto origin_column = block->get_by_position(column_position).column; result_column = filter_column_with_selector(origin_column, selector, count); return Status::OK(); } DataTypePtr execute_type(const Block* block) const override { DCHECK(_open_finished || block == nullptr); - return block->get_by_position(_column_id + _gap).type; + const int column_position = _get_column_position_without_context(block); + if (column_position < 0 || column_position >= block->columns()) { + throw doris::Exception( + ErrorCode::INTERNAL_ERROR, + "input block not contain column ref {}, column_id={}, gap={}, block={}", + _column_name, _column_id, _gap.load(), block->dump_structure()); + } + return block->get_by_position(column_position).type; } bool is_constant() const override { return false; } @@ -75,11 +91,11 @@ class VColumnRef final : public VExpr { const std::string& expr_name() const override { return _column_name; } - void set_gap(int gap) { - if (_gap == 0) { - _gap = gap; - } - } + void set_gap(int gap) { _gap = gap; } + + int get_gap() const { return _gap.load(); } + + bool has_gap() const { return _gap.load() >= 0; } Status clone_node(VExprSPtr* cloned_expr) const override { DORIS_CHECK(cloned_expr != nullptr); @@ -104,8 +120,36 @@ class VColumnRef final : public VExpr { double execute_cost() const override { return 0.0; } private: + int _get_column_position(VExprContext* context, const Block* block) const { + const auto resolve_result = + context->lambda_execution_context().resolve_column_position(_column_name); + if (resolve_result.found) { + return resolve_result.column_position; + } + if (resolve_result.searched_named_scope) { + return -1; + } + return _get_column_position_without_context(block); + } + + int _get_column_position_without_context(const Block* block) const { + if (_gap.load() == -1) { + return _find_column_position_by_name(block); + } + return _column_id + _gap.load(); + } + + int _find_column_position_by_name(const Block* block) const { + for (int position = block->columns() - 1; position >= 0; --position) { + if (block->get_by_position(position).name == _column_name) { + return position; + } + } + return -1; + } + int _column_id; - std::atomic _gap = 0; + std::atomic _gap = -1; std::string _column_name; }; } // namespace doris diff --git a/be/src/exprs/vcompound_pred.h b/be/src/exprs/vcompound_pred.h index d437f975ad147b..95a32ce40025da 100644 --- a/be/src/exprs/vcompound_pred.h +++ b/be/src/exprs/vcompound_pred.h @@ -127,6 +127,14 @@ class VCompoundPred : public VectorizedFnCall { } } + bool is_safe_to_execute_on_selected_rows() const override { + // Boolean composition introduces no data-dependent failure of its own. Reuse the generic + // child walk so AND/OR remain eligible only when every nested expression is independently + // safe; applying VectorizedFnCall's scalar-function allowlist to this structural node would + // incorrectly disable selected-row execution for otherwise safe predicates. + return VExpr::is_safe_to_execute_on_selected_rows(); + } + ZoneMapFilterResult evaluate_dictionary_filter( const DictionaryEvalContext& ctx) const override { switch (_op) { diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index b6e5c13d111756..48b7ed3eaff70e 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -24,9 +24,11 @@ #include #include +#include #include #include +#include "common/compare.h" #include "common/config.h" #include "common/exception.h" #include "common/logging.h" @@ -82,6 +84,82 @@ const static std::set DISTANCE_FUNCS = {L2DistanceApproximate::name const static std::set OPS_FOR_ANN_RANGE_SEARCH = { TExprOpcode::GE, TExprOpcode::LE, TExprOpcode::LE, TExprOpcode::GT, TExprOpcode::LT}; +namespace { + +enum class RawComparisonOp : uint8_t { EQ, NE, LT, LE, GT, GE }; + +std::optional raw_comparison_op(std::string_view function_name, bool reverse) { + RawComparisonOp op; + if (function_name == "eq") { + op = RawComparisonOp::EQ; + } else if (function_name == "ne") { + op = RawComparisonOp::NE; + } else if (function_name == "lt") { + op = RawComparisonOp::LT; + } else if (function_name == "le") { + op = RawComparisonOp::LE; + } else if (function_name == "gt") { + op = RawComparisonOp::GT; + } else if (function_name == "ge") { + op = RawComparisonOp::GE; + } else { + return std::nullopt; + } + if (!reverse || op == RawComparisonOp::EQ || op == RawComparisonOp::NE) { + return op; + } + switch (op) { + case RawComparisonOp::LT: + return RawComparisonOp::GT; + case RawComparisonOp::LE: + return RawComparisonOp::GE; + case RawComparisonOp::GT: + return RawComparisonOp::LT; + case RawComparisonOp::GE: + return RawComparisonOp::LE; + case RawComparisonOp::EQ: + case RawComparisonOp::NE: + break; + } + __builtin_unreachable(); +} + +template +void execute_raw_comparison(const uint8_t* values, size_t num_values, const Field& literal, + RawComparisonOp op, uint8_t* matches) { + const T rhs = literal.get(); + for (size_t row = 0; row < num_values; ++row) { + if (matches[row] == 0) { + continue; + } + const T lhs = unaligned_load(values + row * sizeof(T)); + bool keep = false; + switch (op) { + case RawComparisonOp::EQ: + keep = Compare::equal(lhs, rhs); + break; + case RawComparisonOp::NE: + keep = Compare::not_equal(lhs, rhs); + break; + case RawComparisonOp::LT: + keep = Compare::less(lhs, rhs); + break; + case RawComparisonOp::LE: + keep = Compare::less_equal(lhs, rhs); + break; + case RawComparisonOp::GT: + keep = Compare::greater(lhs, rhs); + break; + case RawComparisonOp::GE: + keep = Compare::greater_equal(lhs, rhs); + break; + } + matches[row] = static_cast(keep); + } +} + +} // namespace + VectorizedFnCall::VectorizedFnCall(const TExprNode& node) : VExpr(node) { _function_name = _fn.name.function_name; } @@ -339,6 +417,71 @@ Status VectorizedFnCall::execute_column_impl(VExprContext* context, const Block* return _do_execute(context, block, selector, count, result_column, nullptr); } +bool VectorizedFnCall::can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + if (data_type == nullptr || !raw_comparison_op(_function_name, false).has_value()) { + return false; + } + auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + if (!slot_literal.has_value() || slot_literal->slot_index != column_id || + slot_literal->literal.is_null()) { + return false; + } + const auto raw_type = remove_nullable(data_type); + if (!remove_nullable(slot_literal->slot_type)->equals(*raw_type) || + !remove_nullable(slot_literal->literal_type)->equals(*raw_type)) { + return false; + } + const auto primitive_type = raw_type->get_primitive_type(); + return primitive_type == TYPE_INT || primitive_type == TYPE_BIGINT || + primitive_type == TYPE_FLOAT || primitive_type == TYPE_DOUBLE; +} + +Status VectorizedFnCall::execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const { + if (!can_execute_on_raw_fixed_values(data_type, column_id)) { + return Status::NotSupported("Expression {} cannot evaluate raw fixed-width values", + expr_name()); + } + DORIS_CHECK(values != nullptr || num_values == 0); + DORIS_CHECK(matches != nullptr || num_values == 0); + const auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + DORIS_CHECK(slot_literal.has_value()); + const auto op = raw_comparison_op(_function_name, slot_literal->literal_on_left); + DORIS_CHECK(op.has_value()); + const auto primitive_type = remove_nullable(data_type)->get_primitive_type(); + const size_t expected_width = primitive_type == TYPE_INT || primitive_type == TYPE_FLOAT + ? sizeof(uint32_t) + : sizeof(uint64_t); + if (value_width != expected_width) { + return Status::Corruption("Raw expression width {} does not match expected {}", value_width, + expected_width); + } + switch (primitive_type) { + case TYPE_INT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_BIGINT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_FLOAT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_DOUBLE: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + default: + __builtin_unreachable(); + } + return Status::OK(); +} + const std::string& VectorizedFnCall::expr_name() const { return _expr_name; } @@ -386,8 +529,12 @@ bool VectorizedFnCall::is_deterministic() const { } bool VectorizedFnCall::is_safe_to_execute_on_selected_rows() const { - static const std::set ERROR_PRESERVING_FUNCTIONS = {"assert_true"}; - return !ERROR_PRESERVING_FUNCTIONS.contains(_function_name) && + static const std::set TOTAL_PREDICATE_FUNCTIONS = { + "eq", "ne", "lt", "le", "gt", "ge", "in", "not_in", "is_null_pred", "is_not_null_pred"}; + // Selected-row execution may hide data-dependent errors in rows rejected by an earlier + // predicate. Keep function calls unsafe by default and opt in only operations that are total + // for their input domain; child checks then reject expressions such as gt(mod(x, -1), 0). + return TOTAL_PREDICATE_FUNCTIONS.contains(_function_name) && VExpr::is_safe_to_execute_on_selected_rows(); } diff --git a/be/src/exprs/vectorized_fn_call.h b/be/src/exprs/vectorized_fn_call.h index 8343242ce314fd..f4e0dcb8f527fe 100644 --- a/be/src/exprs/vectorized_fn_call.h +++ b/be/src/exprs/vectorized_fn_call.h @@ -57,6 +57,11 @@ class VectorizedFnCall : public VExpr { Status execute_runtime_filter(VExprContext* context, const Block* block, const uint8_t* __restrict filter, size_t count, ColumnPtr& result_column, ColumnPtr* arg_column) const override; + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override; + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const override; Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; bool can_evaluate_zonemap_filter() const override; diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index f54af6f7819acb..77520377af6f93 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -174,6 +174,19 @@ class VExpr { uint8_t* __restrict result_filter_data, size_t rows, bool accept_null, bool* can_filter_all) const; + // Raw fixed-width evaluation is an optional expression capability used before a storage reader + // materializes a column. `matches` is ANDed in place; callers handle NULL rows separately + // because raw value streams contain only non-NULL payloads. + virtual bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + return false; + } + virtual Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, const DataTypePtr& data_type, + int column_id, uint8_t* matches) const { + return Status::NotSupported("{} cannot evaluate raw fixed-width values", expr_name()); + } + // `is_blockable` means this expr will be blocked in `execute` (e.g. AI Function, Remote Function) [[nodiscard]] virtual bool is_blockable() const { return std::any_of(_children.begin(), _children.end(), diff --git a/be/src/exprs/vexpr_context.cpp b/be/src/exprs/vexpr_context.cpp index 91ed2305b38218..13a2e0b3d445f8 100644 --- a/be/src/exprs/vexpr_context.cpp +++ b/be/src/exprs/vexpr_context.cpp @@ -19,7 +19,9 @@ #include #include +#include #include +#include #include "common/compiler_util.h" // IWYU pragma: keep #include "common/exception.h" @@ -32,6 +34,7 @@ #include "core/column/column_const.h" #include "exec/common/util.hpp" #include "exprs/function_context.h" +#include "exprs/lambda_function/lambda_execution_context.h" #include "exprs/vexpr.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" @@ -45,6 +48,8 @@ class RowDescriptor; namespace doris { +VExprContext::VExprContext(VExprSPtr expr) : _root(std::move(expr)) {} + VExprContext::~VExprContext() { // In runtime filter, only create expr context to get expr root, will not call // prepare or open, so that it is not need to call close. And call close may core @@ -59,6 +64,13 @@ VExprContext::~VExprContext() { } } +LambdaExecutionContext& VExprContext::lambda_execution_context() { + if (!_lambda_execution_context) { + _lambda_execution_context = std::make_unique(); + } + return *_lambda_execution_context; +} + Status VExprContext::execute(Block* block, int* result_column_id) { Status st; RETURN_IF_CATCH_EXCEPTION({ diff --git a/be/src/exprs/vexpr_context.h b/be/src/exprs/vexpr_context.h index 2850149fbae2b1..5bfd77f923e4d4 100644 --- a/be/src/exprs/vexpr_context.h +++ b/be/src/exprs/vexpr_context.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,7 @@ class ColumnIterator; namespace doris { class ScoreRuntime; +class LambdaExecutionContext; using ScoreRuntimeSPtr = std::shared_ptr; class IndexExecContext { @@ -242,7 +244,7 @@ class VExprContext { ENABLE_FACTORY_CREATOR(VExprContext); public: - VExprContext(VExprSPtr expr) : _root(std::move(expr)) {} + VExprContext(VExprSPtr expr); ~VExprContext(); [[nodiscard]] Status prepare(RuntimeState* state, const RowDescriptor& row_desc); [[nodiscard]] Status open(RuntimeState* state); @@ -266,6 +268,8 @@ class VExprContext { std::shared_ptr get_index_context() const { return _index_context; } + LambdaExecutionContext& lambda_execution_context(); + /// Creates a FunctionContext, and returns the index that's passed to fn_context() to /// retrieve the created context. Exprs that need a FunctionContext should call this in /// Prepare() and save the returned index. 'varargs_buffer_size', if specified, is the @@ -350,36 +354,9 @@ class VExprContext { void clone_fn_contexts(VExprContext* other); - VExprContext& operator=(const VExprContext& other) { - if (this == &other) { - return *this; - } + VExprContext& operator=(const VExprContext& other) = delete; - _root = other._root; - _is_clone = other._is_clone; - _prepared = other._prepared; - _opened = other._opened; - - for (const auto& fn : other._fn_contexts) { - _fn_contexts.emplace_back(fn->clone()); - } - - _last_result_column_id = other._last_result_column_id; - _depth_num = other._depth_num; - return *this; - } - - VExprContext& operator=(VExprContext&& other) { - _root = other._root; - other._root = nullptr; - _is_clone = other._is_clone; - _prepared = other._prepared; - _opened = other._opened; - _fn_contexts = std::move(other._fn_contexts); - _last_result_column_id = other._last_result_column_id; - _depth_num = other._depth_num; - return *this; - } + VExprContext& operator=(VExprContext&& other) = delete; [[nodiscard]] static size_t get_memory_usage(const VExprContextSPtrs& contexts) { size_t usage = 0; @@ -437,6 +414,8 @@ class VExprContext { segment_v2::AnnRangeSearchRuntime _ann_range_search_runtime; bool _suitable_for_ann_index = true; + std::unique_ptr _lambda_execution_context; + std::unique_ptr _rf_selectivity = std::make_unique(); }; diff --git a/be/src/exprs/vlambda_function_call_expr.h b/be/src/exprs/vlambda_function_call_expr.h index 64b8118ed600c8..a3764fc13fdf05 100644 --- a/be/src/exprs/vlambda_function_call_expr.h +++ b/be/src/exprs/vlambda_function_call_expr.h @@ -50,7 +50,7 @@ class VLambdaFunctionCallExpr : public VExpr { return Status::InternalError("Lambda Function {} is not implemented.", _fn.name.function_name); } - RETURN_IF_ERROR(_lambda_function->prepare(state)); + RETURN_IF_ERROR(_lambda_function->prepare(state, _children)); _prepare_finished = true; return Status::OK(); } diff --git a/be/src/exprs/vlambda_function_expr.h b/be/src/exprs/vlambda_function_expr.h index 028489cb8821a7..9b8f7474287830 100644 --- a/be/src/exprs/vlambda_function_expr.h +++ b/be/src/exprs/vlambda_function_expr.h @@ -16,6 +16,9 @@ // under the License. #pragma once +#include +#include + #include "common/global_types.h" #include "exprs/function/function.h" #include "exprs/vexpr.h" @@ -25,7 +28,10 @@ class VLambdaFunctionExpr final : public VExpr { ENABLE_FACTORY_CREATOR(VLambdaFunctionExpr); public: - VLambdaFunctionExpr(const TExprNode& node) : VExpr(node) {} + VLambdaFunctionExpr(const TExprNode& node) + : VExpr(node), + _has_argument_names(node.__isset.lambda_argument_names), + _argument_names(node.lambda_argument_names) {} ~VLambdaFunctionExpr() override = default; Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override { @@ -58,9 +64,15 @@ class VLambdaFunctionExpr final : public VExpr { const std::string& expr_name() const override { return _expr_name; } + bool has_argument_names() const { return _has_argument_names; } + + const std::vector& argument_names() const { return _argument_names; } + uint64_t get_digest(uint64_t seed) const override { return 0; } private: const std::string _expr_name = "vlambda_function_expr"; + const bool _has_argument_names; + const std::vector _argument_names; }; } // namespace doris diff --git a/be/src/exprs/vslot_ref.h b/be/src/exprs/vslot_ref.h index a67bdc1953cd0a..680d8b367617a4 100644 --- a/be/src/exprs/vslot_ref.h +++ b/be/src/exprs/vslot_ref.h @@ -42,6 +42,7 @@ class VSlotRef : public VExpr { #ifdef BE_TEST VSlotRef() = default; void set_slot_id(int slot_id) { _slot_id = slot_id; } + void set_column_name(const std::string* column_name) { _column_name = column_name; } #endif void set_column_id(int column_id) { _column_id = column_id; } Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 28ef930057f3a2..728595bdad13a7 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -34,6 +34,7 @@ #include "exprs/vexpr.h" #include "exprs/vslot_ref.h" #include "exprs/vtopn_pred.h" +#include "util/url_coding.h" // IWYU pragma: no_include #include // IWYU pragma: keep @@ -105,6 +106,35 @@ #include "util/unaligned.h" namespace doris { +static Status build_orc_initial_default_column( + const std::optional& metadata, + const DataTypePtr& type, size_t rows, ColumnPtr* column) { + DORIS_CHECK(column != nullptr); + if (!metadata.has_value()) { + *column = nullptr; + return Status::OK(); + } + const auto nested_type = remove_nullable(type); + Field value; + if (metadata->is_base64 || nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(metadata->value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg nested initial default"); + } + value = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + // StringView borrows decoded for payloads longer than 12 bytes. Build the owning column + // before the decode buffer leaves scope (UUID/FIXED defaults routinely exceed that size). + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); + } else { + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(metadata->value, value)); + } + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); +} + class RuntimeState; namespace io { struct IOContext; @@ -2145,15 +2175,28 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, for (int missing_field : missing_fields) { ColumnPtr& doris_field = doris_struct.get_column_ptr(missing_field); - if (!doris_field->is_nullable()) { + const auto& table_column_name = doris_struct_type->get_name_by_position(missing_field); + const auto& doris_type = doris_struct_type->get_element(missing_field); + ColumnPtr initial_default; + RETURN_IF_ERROR(build_orc_initial_default_column( + root_node->children_initial_default_value(table_column_name), doris_type, + num_values, &initial_default)); + if (initial_default.get() != nullptr) { + // ORC projection may synthesize a missing nested field, but its Iceberg initial + // default remains the logical value for every row in the older file. + auto mutable_field = IColumn::mutate(std::move(doris_field)); + mutable_field->insert_range_from(*initial_default, 0, num_values); + doris_field = std::move(mutable_field); + } else if (!doris_field->is_nullable()) { return Status::InternalError( "Child field of '{}' is not nullable, but is missing in orc file", col_name); + } else { + auto mutable_field = IColumn::mutate(std::move(doris_field)); + reinterpret_cast(mutable_field.get()) + ->insert_many_defaults(num_values); + doris_field = std::move(mutable_field); } - auto mutable_field = IColumn::mutate(std::move(doris_field)); - reinterpret_cast(mutable_field.get()) - ->insert_many_defaults(num_values); - doris_field = std::move(mutable_field); } for (auto read_field : read_fields) { diff --git a/be/src/format/parquet/vparquet_column_reader.cpp b/be/src/format/parquet/vparquet_column_reader.cpp index 2726044c193b4c..53e44babbd8297 100644 --- a/be/src/format/parquet/vparquet_column_reader.cpp +++ b/be/src/format/parquet/vparquet_column_reader.cpp @@ -40,8 +40,38 @@ #include "format/parquet/vparquet_column_chunk_reader.h" #include "io/fs/tracing_file_reader.h" #include "runtime/runtime_profile.h" +#include "util/url_coding.h" namespace doris { +static Status build_initial_default_column( + const std::optional& metadata, + const DataTypePtr& type, size_t rows, ColumnPtr* column) { + DORIS_CHECK(column != nullptr); + if (!metadata.has_value()) { + *column = nullptr; + return Status::OK(); + } + const auto nested_type = remove_nullable(type); + Field value; + if (metadata->is_base64 || nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(metadata->value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg nested initial default"); + } + value = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + // StringView borrows decoded for payloads longer than 12 bytes. Build the owning column + // before the decode buffer leaves scope (UUID/FIXED defaults routinely exceed that size). + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); + } else { + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(metadata->value, value)); + } + *column = type->create_column_const(rows, value)->convert_to_full_column_if_const(); + return Status::OK(); +} + static void fill_struct_null_map(FieldSchema* field, NullMap& null_map, const std::vector& rep_levels, const std::vector& def_levels) { @@ -990,11 +1020,21 @@ Status StructColumnReader::read_column_data( for (auto idx : missing_column_idxs) { auto& doris_field = doris_struct.get_column_ptr(idx); auto& doris_type = doris_struct_type->get_element(idx); - DCHECK(doris_type->is_nullable()); - doris_field = IColumn::mutate(std::move(doris_field)); - auto mutable_column = doris_field->assert_mutable(); - auto* nullable_column = static_cast(mutable_column.get()); - nullable_column->insert_many_defaults(missing_column_sz); + auto mutable_field = IColumn::mutate(std::move(doris_field)); + ColumnPtr initial_default; + RETURN_IF_ERROR(build_initial_default_column( + root_node->children_initial_default_value(doris_struct_type->get_element_name(idx)), + doris_type, missing_column_sz, &initial_default)); + if (initial_default.get() != nullptr) { + // Iceberg initial defaults are logical row values, including for nested fields absent + // from the physical file; append them instead of the type's generic NULL/default. + mutable_field->insert_range_from(*initial_default, 0, missing_column_sz); + } else { + DCHECK(doris_type->is_nullable()); + static_cast(mutable_field.get()) + ->insert_many_defaults(missing_column_sz); + } + doris_field = std::move(mutable_field); } if (null_map_ptr != nullptr) { diff --git a/be/src/format/table/deletion_vector.h b/be/src/format/table/deletion_vector.h index 2c89771b2e6b0e..12ec7dbc6ebc80 100644 --- a/be/src/format/table/deletion_vector.h +++ b/be/src/format/table/deletion_vector.h @@ -17,10 +17,22 @@ #pragma once +#include +#include + #include "roaring/roaring64map.hh" namespace doris { +// Doris materializes an Iceberg deletion vector as one buffer. This is an implementation limit, +// not an Iceberg format limit. +inline constexpr int64_t MAX_ICEBERG_DELETION_VECTOR_BYTES = 1L << 30; +inline constexpr size_t ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES = 12; + +// Paimon v1 uses a run-optimized 32-bit Roaring bitmap whose maximum serialized size is below this +// limit. Keep its guard independent from the Iceberg capability limit. +inline constexpr int64_t MAX_PAIMON_DELETION_VECTOR_BYTES = 1L << 30; + // A deletion vector is already a bitmap on the wire. Keep decoded DVs compressed in the // query-local cache instead of expanding every set bit into an int64_t. Position delete files use // a different representation because their input is a stream of (file_path, row_position) rows. diff --git a/be/src/format/table/deletion_vector_reader.cpp b/be/src/format/table/deletion_vector_reader.cpp index 65ce2831a0943a..8e8404839be9a8 100644 --- a/be/src/format/table/deletion_vector_reader.cpp +++ b/be/src/format/table/deletion_vector_reader.cpp @@ -17,11 +17,77 @@ #include "format/table/deletion_vector_reader.h" +#include + #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "util/block_compression.h" namespace doris { +namespace { + +constexpr int64_t ICEBERG_DELETION_VECTOR_MIN_BYTES = + static_cast(ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES); +constexpr int64_t PAIMON_DELETION_VECTOR_MIN_BYTES = 8; +constexpr int64_t PAIMON_LENGTH_PREFIX_BYTES = 4; + +enum class DeletionVectorSizeLimitStatus { + DATA_QUALITY, + NOT_SUPPORTED, +}; + +Status validate_deletion_vector_read_range(const char* description, int64_t offset, int64_t size, + int64_t min_size, int64_t max_size, + DeletionVectorSizeLimitStatus size_limit_status, + size_t& bytes_read) { + if (offset < 0) { + return Status::DataQualityError("{} offset must be non-negative: {}", description, offset); + } + if (size < min_size) { + return Status::DataQualityError("{} size too small: {}, minimum: {}", description, size, + min_size); + } + if (size > max_size) { + if (size_limit_status == DeletionVectorSizeLimitStatus::NOT_SUPPORTED) { + return Status::NotSupported("{} size exceeds Doris supported limit: {}, limit: {}", + description, size, max_size); + } + return Status::DataQualityError("{} size exceeds limit: {}, limit: {}", description, size, + max_size); + } + if (offset > std::numeric_limits::max() - size) { + return Status::DataQualityError("{} offset plus size overflows: offset {}, size {}", + description, offset, size); + } + bytes_read = static_cast(size); + return Status::OK(); +} + +} // namespace + +Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t size, + size_t& bytes_read) { + return validate_deletion_vector_read_range( + "Iceberg deletion vector", offset, size, ICEBERG_DELETION_VECTOR_MIN_BYTES, + MAX_ICEBERG_DELETION_VECTOR_BYTES, DeletionVectorSizeLimitStatus::NOT_SUPPORTED, + bytes_read); +} + +Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t length, + size_t& bytes_read) { + if (length < 0) { + return Status::DataQualityError("Paimon deletion vector length must be non-negative: {}", + length); + } + if (length > std::numeric_limits::max() - PAIMON_LENGTH_PREFIX_BYTES) { + return Status::DataQualityError("Paimon deletion vector length overflows: {}", length); + } + return validate_deletion_vector_read_range( + "Paimon deletion vector", offset, length + PAIMON_LENGTH_PREFIX_BYTES, + PAIMON_DELETION_VECTOR_MIN_BYTES, MAX_PAIMON_DELETION_VECTOR_BYTES, + DeletionVectorSizeLimitStatus::DATA_QUALITY, bytes_read); +} + DeletionVectorReader::~DeletionVectorReader() { // The file reader may retain the child IOContext. Destroy it before merging and before the // child statistics storage goes away. @@ -60,11 +126,25 @@ Status DeletionVectorReader::open() { return Status::OK(); } + if (_desc.start_offset < 0 || _desc.size < 0) { + return Status::DataQualityError( + "Deletion vector range must be non-negative: path {}, offset {}, size {}", + _desc.path, _desc.start_offset, _desc.size); + } + _init_system_properties(); _init_file_description(); RETURN_IF_ERROR(_create_file_reader()); - _file_size = _file_reader->size(); + const size_t file_size = _file_reader->size(); + const size_t start_offset = static_cast(_desc.start_offset); + const size_t range_size = static_cast(_desc.size); + if (start_offset > file_size || range_size > file_size - start_offset) { + return Status::DataQualityError( + "Deletion vector range exceeds file size: path {}, offset {}, size {}, file size " + "{}", + _desc.path, _desc.start_offset, _desc.size, file_size); + } _is_opened = true; return Status::OK(); } diff --git a/be/src/format/table/deletion_vector_reader.h b/be/src/format/table/deletion_vector_reader.h index 55f0099b6feedf..c187e2871687fe 100644 --- a/be/src/format/table/deletion_vector_reader.h +++ b/be/src/format/table/deletion_vector_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -37,6 +38,12 @@ struct IOContext; } // namespace io namespace doris { +Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t size, + size_t& bytes_read); + +Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t length, + size_t& bytes_read); + struct DeleteFileDesc { enum class Format { PAIMON, @@ -111,7 +118,6 @@ class DeletionVectorReader { io::FileSystemProperties _system_properties; io::FileDescription _file_description; io::FileReaderSPtr _file_reader; - int64_t _file_size = 0; bool _is_opened = false; }; } // namespace doris diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.cpp b/be/src/format/table/iceberg_delete_file_reader_helper.cpp index 9f4c0482e2abcc..f8828e402b03e9 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.cpp +++ b/be/src/format/table/iceberg_delete_file_reader_helper.cpp @@ -48,6 +48,7 @@ #include "runtime/runtime_state.h" #include "storage/predicate/column_predicate.h" #include "util/debug_points.h" +#include "util/hash_util.hpp" namespace doris { @@ -55,6 +56,7 @@ namespace { constexpr const char* ICEBERG_FILE_PATH = "file_path"; constexpr const char* ICEBERG_ROW_POS = "pos"; +constexpr size_t ICEBERG_DELETION_VECTOR_MIN_BYTES = 12; const std::vector DELETE_COL_NAMES {ICEBERG_FILE_PATH, ICEBERG_ROW_POS}; std::unordered_map DELETE_COL_NAME_TO_BLOCK_IDX = {{ICEBERG_FILE_PATH, 0}, @@ -177,14 +179,14 @@ Status decode_deletion_vector_buffer(const char* buf, size_t buffer_size, if (buf == nullptr || rows_to_delete == nullptr) { return Status::InvalidArgument("invalid deletion vector decode arguments"); } - if (buffer_size < 12) { + if (buffer_size < ICEBERG_DELETION_VECTOR_MIN_BYTES) { return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); } - auto total_length = BigEndian::Load32(buf); - if (total_length + 8 != buffer_size) { + const uint32_t total_length = BigEndian::Load32(buf); + if (static_cast(total_length) + 8 != buffer_size) { return Status::DataQualityError("Deletion vector length mismatch, expected: {}, actual: {}", - total_length + 8, buffer_size); + static_cast(total_length) + 8, buffer_size); } constexpr static char MAGIC_NUMBER[] = {'\xD1', '\xD3', '\x39', '\x64'}; @@ -192,8 +194,17 @@ Status decode_deletion_vector_buffer(const char* buf, size_t buffer_size, return Status::DataQualityError("Deletion vector magic number mismatch"); } + const uint32_t expected_crc = BigEndian::Load32(buf + sizeof(total_length) + total_length); + const uint32_t actual_crc = + HashUtil::zlib_crc_hash(buf + sizeof(total_length), total_length, 0); + if (actual_crc != expected_crc) { + return Status::DataQualityError("Deletion vector CRC32 mismatch, expected: {}, actual: {}", + expected_crc, actual_crc); + } + try { - *rows_to_delete |= roaring::Roaring64Map::readSafe(buf + 8, buffer_size - 12); + *rows_to_delete |= roaring::Roaring64Map::readSafe( + buf + 8, buffer_size - ICEBERG_DELETION_VECTOR_MIN_BYTES); } catch (const std::runtime_error& e) { return Status::DataQualityError("Decode roaring bitmap failed, {}", e.what()); } @@ -245,6 +256,18 @@ std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file delete_file.content_size_in_bytes); } +Status validate_iceberg_deletion_vector_descriptor(const TIcebergDeleteFileDesc& delete_file, + size_t& bytes_read) { + if (!delete_file.__isset.path || !delete_file.__isset.content_offset || + !delete_file.__isset.content_size_in_bytes) { + return Status::DataQualityError( + "Iceberg deletion vector descriptor misses " + "path/content_offset/content_size_in_bytes"); + } + return validate_iceberg_deletion_vector_read_range( + delete_file.content_offset, delete_file.content_size_in_bytes, bytes_read); +} + Status read_iceberg_position_delete_file(const TIcebergDeleteFileDesc& delete_file, const IcebergDeleteFileReaderOptions& options, IcebergPositionDeleteVisitor* visitor) { @@ -318,9 +341,8 @@ Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file, options.io_ctx == nullptr || rows_to_delete == nullptr) { return Status::InvalidArgument("invalid deletion vector reader options"); } - if (!delete_file.__isset.content_offset || !delete_file.__isset.content_size_in_bytes) { - return Status::InternalError("Deletion vector is missing content offset or length"); - } + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_iceberg_deletion_vector_descriptor(delete_file, bytes_read)); DBUG_EXECUTE_IF("IcebergDeleteFileReader.read_deletion_vector.io_error", { return Status::IOError("injected Iceberg deletion vector read failure"); }); DBUG_EXECUTE_IF("IcebergDeleteFileReader.read_deletion_vector.should_stop", @@ -333,18 +355,19 @@ Status read_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file, delete_range.start_offset = delete_file.content_offset; delete_range.size = delete_file.content_size_in_bytes; + // Iceberg v3 deletion-vector-v1 blobs are uncompressed and metadata provides the exact range. + // Parse the Puffin footer first if future blob types or compression codecs are supported. DeletionVectorReader dv_reader(options.state, options.profile, *options.scan_params, delete_range, options.io_ctx); RETURN_IF_ERROR(dv_reader.open()); - std::vector buf(delete_range.size); - const auto read_status = dv_reader.read_at(delete_range.start_offset, - {buf.data(), cast_set(delete_range.size)}); + std::vector buf(bytes_read); + const auto read_status = dv_reader.read_at(delete_range.start_offset, {buf.data(), bytes_read}); if (options.deletion_vector_file_cache_stats != nullptr) { options.deletion_vector_file_cache_stats->merge_from(dv_reader.file_cache_statistics()); } RETURN_IF_ERROR(read_status); - return decode_deletion_vector_buffer(buf.data(), delete_range.size, rows_to_delete); + return decode_deletion_vector_buffer(buf.data(), bytes_read, rows_to_delete); } Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.h b/be/src/format/table/iceberg_delete_file_reader_helper.h index 0b4f601c0e8021..adc0ef196f4b75 100644 --- a/be/src/format/table/iceberg_delete_file_reader_helper.h +++ b/be/src/format/table/iceberg_delete_file_reader_helper.h @@ -74,6 +74,9 @@ bool is_iceberg_deletion_vector(const TIcebergDeleteFileDesc& delete_file); std::string build_iceberg_deletion_vector_cache_key(const std::string& data_file_path, const TIcebergDeleteFileDesc& delete_file); +Status validate_iceberg_deletion_vector_descriptor(const TIcebergDeleteFileDesc& delete_file, + size_t& bytes_read); + Status decode_iceberg_deletion_vector_buffer(const char* buf, size_t buffer_size, DeletionVector* rows_to_delete); diff --git a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp index c6e97465e2dd44..3721798c7f1e63 100644 --- a/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format/table/iceberg_position_delete_sys_table_reader.cpp @@ -37,6 +37,7 @@ #include "format/orc/vorc_reader.h" #include "format/parquet/schema_desc.h" #include "format/parquet/vparquet_reader.h" +#include "format/table/iceberg_scan_semantics.h" #include "format/table/parquet_utils.h" #include "format/table/table_schema_change_helper.h" #include "runtime/runtime_state.h" @@ -99,15 +100,7 @@ const ColumnInt64* get_int64_column(const Block& block, const std::string& name) return check_and_get_column(block.get_by_position(pos).column.get()); } -const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& field_ptr) { - if (!field_ptr.__isset.field_ptr || field_ptr.field_ptr == nullptr) { - return nullptr; - } - return field_ptr.field_ptr.get(); -} - -const schema::external::TField* find_current_schema_field(const TFileScanRangeParams* params, - const std::string& name) { +const schema::external::TSchema* find_current_schema(const TFileScanRangeParams* params) { if (params == nullptr || !params->__isset.history_schema_info || params->history_schema_info.empty()) { return nullptr; @@ -121,16 +114,7 @@ const schema::external::TField* find_current_schema_field(const TFileScanRangePa } } } - if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { - return nullptr; - } - for (const auto& field_ptr : schema->root_field.fields) { - const auto* field = get_field_ptr(field_ptr); - if (field != nullptr && field->__isset.name && field->name == name) { - return field; - } - } - return nullptr; + return schema; } template @@ -248,13 +232,24 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { &_state->timezone_obj(), _io_ctx, _state, _meta_cache); const FieldDescriptor* schema = nullptr; - int row_index = -1; + std::shared_ptr mapped_file_schema; if (row_requested) { RETURN_IF_ERROR(parquet_reader->get_file_metadata_schema(&schema)); DORIS_CHECK(schema != nullptr); - row_index = schema->get_column_index(kRowColumn); + const auto* table_schema = find_current_schema(_range_params); + if (table_schema == nullptr || !table_schema->__isset.root_field) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); + } + // Position-delete mapping mode is file-wide: file_path/pos IDs must prevent an + // ID-less physical row from being rebound by name in either reader generation. + RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil:: + by_parquet_field_id_with_name_mapping( + table_schema->root_field, *schema, mapped_file_schema, + supports_iceberg_scan_semantics_v1(_range_params))); } - const bool read_row = row_requested && row_index >= 0; + const bool read_row = + row_requested && mapped_file_schema->children_column_exists(kRowColumn); _init_read_columns(read_row); std::vector read_column_names; read_column_names.reserve(_read_columns.size()); @@ -270,18 +265,10 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { pctx.range = &_range; pctx.filter_groups = false; if (read_row) { - const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); - if (table_row_field == nullptr) { - return Status::InternalError( - "Iceberg position delete system table row schema is missing"); - } - const auto* file_row_field = schema->get_column(static_cast(row_index)); - std::shared_ptr row_node; - RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil:: - by_parquet_field_id_with_name_mapping( - *table_row_field, *file_row_field, row_node)); auto root_node = create_position_delete_root_node(_read_columns); - root_node->add_children(kRowColumn, file_row_field->name, row_node); + root_node->add_children(kRowColumn, + mapped_file_schema->children_file_column_name(kRowColumn), + mapped_file_schema->get_children_node(kRowColumn)); pctx.table_info_node = std::move(root_node); } RETURN_IF_ERROR(static_cast(parquet_reader.get())->init_reader(&pctx)); @@ -294,19 +281,25 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { OrcReader::create_unique(_profile, _state, *_range_params, _range, _batch_size, _state->timezone(), _io_ctx, _meta_cache); - const orc::Type* row_type = nullptr; + std::shared_ptr mapped_file_schema; if (row_requested) { const orc::Type* root_type = nullptr; RETURN_IF_ERROR(orc_reader->get_file_type(&root_type)); DORIS_CHECK(root_type != nullptr); - for (uint64_t i = 0; i < root_type->getSubtypeCount(); ++i) { - if (root_type->getFieldName(i) == kRowColumn) { - row_type = root_type->getSubtype(i); - break; - } + const auto* table_schema = find_current_schema(_range_params); + if (table_schema == nullptr || !table_schema->__isset.root_field) { + return Status::InternalError( + "Iceberg position delete system table row schema is missing"); } + // Resolve row against the complete delete-file type so top-level IDs keep ORC in ID + // projection throughout the nested row subtree. + RETURN_IF_ERROR( + TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + table_schema->root_field, root_type, kIcebergOrcAttribute, + mapped_file_schema, supports_iceberg_scan_semantics_v1(_range_params))); } - const bool read_row = row_requested && row_type != nullptr; + const bool read_row = + row_requested && mapped_file_schema->children_column_exists(kRowColumn); _init_read_columns(read_row); std::vector read_column_names; read_column_names.reserve(_read_columns.size()); @@ -321,17 +314,10 @@ Status IcebergPositionDeleteSysTableReader::_init_position_delete_reader() { octx.params = _range_params; octx.range = &_range; if (read_row) { - const auto* table_row_field = find_current_schema_field(_range_params, kRowColumn); - if (table_row_field == nullptr) { - return Status::InternalError( - "Iceberg position delete system table row schema is missing"); - } - std::shared_ptr row_node; - RETURN_IF_ERROR( - TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( - *table_row_field, row_type, kIcebergOrcAttribute, row_node)); auto root_node = create_position_delete_root_node(_read_columns); - root_node->add_children(kRowColumn, kRowColumn, row_node); + root_node->add_children(kRowColumn, + mapped_file_schema->children_file_column_name(kRowColumn), + mapped_file_schema->get_children_node(kRowColumn)); octx.table_info_node = std::move(root_node); } RETURN_IF_ERROR(static_cast(orc_reader.get())->init_reader(&octx)); diff --git a/be/src/format/table/iceberg_reader.cpp b/be/src/format/table/iceberg_reader.cpp index 38bb395120df24..ea6314ae33fbd6 100644 --- a/be/src/format/table/iceberg_reader.cpp +++ b/be/src/format/table/iceberg_reader.cpp @@ -54,6 +54,7 @@ #include "format/table/deletion_vector_reader.h" #include "format/table/iceberg/iceberg_orc_nested_column_utils.h" #include "format/table/iceberg/iceberg_parquet_nested_column_utils.h" +#include "format/table/iceberg_scan_semantics.h" #include "format/table/nested_column_access_helper.h" #include "format/table/table_schema_change_helper.h" #include "runtime/runtime_state.h" @@ -76,6 +77,22 @@ class VExprContext; namespace doris { const std::string IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE = "iceberg.id"; +namespace { + +bool orc_subtree_has_iceberg_id(const orc::Type* type, const std::string& attribute) { + if (type->hasAttributeKey(attribute)) { + return true; + } + for (uint64_t idx = 0; idx < type->getSubtypeCount(); ++idx) { + if (orc_subtree_has_iceberg_id(type->getSubtype(idx), attribute)) { + return true; + } + } + return false; +} + +} // namespace + bool IcebergTableReader::_is_fully_dictionary_encoded( const tparquet::ColumnMetaData& column_metadata) { const auto is_dictionary_encoding = [](tparquet::Encoding::type encoding) { @@ -149,7 +166,7 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { } else { RETURN_IF_ERROR(BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( get_scan_params().history_schema_info.front().root_field, *field_desc, - ctx->table_info_node)); + ctx->table_info_node, supports_iceberg_scan_semantics_v1(&get_scan_params()))); } std::unordered_set partition_col_names; @@ -243,15 +260,22 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { const static std::string EQ_DELETE_PRE = "__equality_delete_column__"; std::unordered_map field_id_to_file_column; bool all_file_columns_have_field_ids = true; + bool any_file_column_has_field_id = false; for (int i = 0; i < field_desc->size(); ++i) { const auto* field_schema = field_desc->get_column(i); if (field_schema) { - field_id_to_file_column[field_schema->field_id] = field_schema; if (field_schema->field_id < 0) { all_file_columns_have_field_ids = false; + } else { + any_file_column_has_field_id = true; + field_id_to_file_column[field_schema->field_id] = field_schema; } } } + const bool use_field_ids_for_hidden_keys = + supports_iceberg_scan_semantics_v1(&get_scan_params()) + ? any_file_column_has_field_id + : all_file_columns_have_field_ids; const auto struct_node = std::dynamic_pointer_cast(ctx->table_info_node); DORIS_CHECK(struct_node != nullptr); @@ -270,26 +294,31 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { } const FieldSchema* file_column = nullptr; - if (!all_file_columns_have_field_ids && struct_node->get_children().contains(old_name) && - struct_node->children_column_exists(old_name)) { - // Iceberg files written without field ids must use schema.name-mapping.default. The - // root schema mapper deliberately switches the whole file to BY_NAME when even one - // top-level field id is absent. Hidden equality keys must make the same choice: a - // different physical column may still carry this key's stale id after migration. - const auto& mapped_name = struct_node->children_file_column_name(old_name); - for (int j = 0; j < field_desc->size(); ++j) { - const auto* candidate = field_desc->get_column(j); - if (candidate != nullptr && candidate->name == mapped_name) { - file_column = candidate; - break; - } - } - DORIS_CHECK(file_column != nullptr); - } else if (all_file_columns_have_field_ids) { + if (use_field_ids_for_hidden_keys) { auto id_it = field_id_to_file_column.find(field_id); if (id_it != field_id_to_file_column.end()) { file_column = id_it->second; } + } else { + std::string table_column_name = old_name; + if (const auto* table_field = _find_current_schema_field(field_id); + table_field != nullptr && table_field->__isset.name) { + table_column_name = table_field->name; + } + // Delete files may retain a pre-rename key name, while StructNode is keyed by the + // current table name. Resolve that name by stable field ID before following BY_NAME. + if (struct_node->get_children().contains(table_column_name) && + struct_node->children_column_exists(table_column_name)) { + const auto& mapped_name = struct_node->children_file_column_name(table_column_name); + for (int j = 0; j < field_desc->size(); ++j) { + const auto* candidate = field_desc->get_column(j); + if (candidate != nullptr && candidate->name == mapped_name) { + file_column = candidate; + break; + } + } + DORIS_CHECK(file_column != nullptr); + } } const std::string file_col_name = file_column == nullptr ? old_name : file_column->name; @@ -364,15 +393,27 @@ ColumnIdResult IcebergParquetReader::_create_column_ids( IcebergParquetNestedColumnUtils::extract_nested_column_ids); }; + // The Iceberg schema-mapping root is a StructNode whose registered children are the real + // table columns. When present, resolve each column by name through it so the column-id set + // stays consistent with the schema-mapping decision (BY_ID or BY_NAME/name-mapping); + // otherwise fall back to matching by Iceberg field id. + const auto* struct_node = + dynamic_cast(table_info_node.get()); + for (const auto* slot : tuple_descriptor->slots()) { const FieldSchema* field_schema = nullptr; - if (table_info_node != nullptr) { - if (table_info_node->children_column_exists(slot->col_name())) { + if (struct_node != nullptr) { + // Synthesized/metadata slots (e.g. the TopN global row-id or the $row_id column) are + // never registered as children, so check membership before querying: calling + // children_column_exists() on an unregistered name DCHECK-aborts in debug builds and + // throws std::out_of_range from .at() in release builds. + if (struct_node->get_children().contains(slot->col_name()) && + struct_node->children_column_exists(slot->col_name())) { // Use the physical child selected by the schema-mapping pass. This keeps partial-id // files in BY_NAME mode from binding a projected column through an unrelated stale // field id. const auto& file_column_name = - table_info_node->children_file_column_name(slot->col_name()); + struct_node->children_file_column_name(slot->col_name()); for (int i = 0; i < field_desc->size(); ++i) { const auto* candidate = field_desc->get_column(i); if (candidate != nullptr && candidate->name == file_column_name) { @@ -485,7 +526,8 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { } else { RETURN_IF_ERROR(BuildTableInfoUtil::by_orc_field_id_with_name_mapping( get_scan_params().history_schema_info.front().root_field, orc_type_ptr, - ICEBERG_ORC_ATTRIBUTE, ctx->table_info_node)); + ICEBERG_ORC_ATTRIBUTE, ctx->table_info_node, + supports_iceberg_scan_semantics_v1(&get_scan_params()))); } std::unordered_set partition_col_names; @@ -581,6 +623,10 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { all_file_columns_have_field_ids = false; } } + const bool use_field_ids_for_hidden_keys = + supports_iceberg_scan_semantics_v1(&get_scan_params()) + ? orc_subtree_has_iceberg_id(orc_type_ptr, ICEBERG_ORC_ATTRIBUTE) + : all_file_columns_have_field_ids; const auto struct_node = std::dynamic_pointer_cast(ctx->table_info_node); DORIS_CHECK(struct_node != nullptr); @@ -597,24 +643,29 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { } const orc::Type* file_column = nullptr; - if (!all_file_columns_have_field_ids && struct_node->get_children().contains(old_name) && - struct_node->children_column_exists(old_name)) { - // Match the root ORC schema mapper's all-or-nothing BY_NAME decision. Accepting a - // matching id in a partial-id file could bind this hidden key to an unrelated stale - // column instead of the current name or historical alias selected by table_info_node. - const auto& mapped_name = struct_node->children_file_column_name(old_name); - for (uint64_t j = 0; j < orc_type_ptr->getSubtypeCount(); ++j) { - if (orc_type_ptr->getFieldName(j) == mapped_name) { - file_column = orc_type_ptr->getSubtype(j); - break; - } - } - DORIS_CHECK(file_column != nullptr); - } else if (all_file_columns_have_field_ids) { + if (use_field_ids_for_hidden_keys) { auto id_it = field_id_to_file_column.find(field_id); if (id_it != field_id_to_file_column.end()) { file_column = id_it->second; } + } else { + std::string table_column_name = old_name; + if (const auto* table_field = _find_current_schema_field(field_id); + table_field != nullptr && table_field->__isset.name) { + table_column_name = table_field->name; + } + // The mapping node uses current names even when an older delete file does not. + if (struct_node->get_children().contains(table_column_name) && + struct_node->children_column_exists(table_column_name)) { + const auto& mapped_name = struct_node->children_file_column_name(table_column_name); + for (uint64_t j = 0; j < orc_type_ptr->getSubtypeCount(); ++j) { + if (orc_type_ptr->getFieldName(j) == mapped_name) { + file_column = orc_type_ptr->getSubtype(j); + break; + } + } + DORIS_CHECK(file_column != nullptr); + } } std::string file_col_name = old_name; @@ -691,15 +742,27 @@ ColumnIdResult IcebergOrcReader::_create_column_ids( IcebergOrcNestedColumnUtils::extract_nested_column_ids); }; + // The Iceberg schema-mapping root is a StructNode whose registered children are the real + // table columns. When present, resolve each column by name through it so the column-id set + // stays consistent with the schema-mapping decision (BY_ID or BY_NAME/name-mapping); + // otherwise fall back to matching by Iceberg field id. + const auto* struct_node = + dynamic_cast(table_info_node.get()); + for (const auto* slot : tuple_descriptor->slots()) { const orc::Type* orc_field = nullptr; - if (table_info_node != nullptr) { - if (table_info_node->children_column_exists(slot->col_name())) { + if (struct_node != nullptr) { + // Synthesized/metadata slots (e.g. the TopN global row-id or the $row_id column) are + // never registered as children, so check membership before querying: calling + // children_column_exists() on an unregistered name DCHECK-aborts in debug builds and + // throws std::out_of_range from .at() in release builds. + if (struct_node->get_children().contains(slot->col_name()) && + struct_node->children_column_exists(slot->col_name())) { // Select the physical child resolved by the shared schema-mapping pass. Hidden // equality keys and projected columns must obey the same BY_NAME decision for // partial-id ORC files. const auto& file_column_name = - table_info_node->children_file_column_name(slot->col_name()); + struct_node->children_file_column_name(slot->col_name()); for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) { if (orc_type->getFieldName(i) == file_column_name) { orc_field = orc_type->getSubtype(i); diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index aee211aee019cd..c9177be2d9a9df 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -40,6 +40,7 @@ #include "format/generic_reader.h" #include "format/table/equality_delete.h" #include "format/table/iceberg_delete_file_reader_helper.h" +#include "format/table/iceberg_scan_semantics.h" #include "format/table/table_schema_change_helper.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" @@ -157,6 +158,33 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { protected: // ---- Hook implementations ---- + Status on_fill_missing_columns(Block* block, size_t rows, + const std::vector& cols) override { + if (!supports_iceberg_scan_semantics_v1(&this->get_scan_params())) { + return BaseReader::on_fill_missing_columns(block, rows, cols); + } + std::vector generic_columns; + for (const auto& name : cols) { + const auto* field = _find_current_schema_field(name); + if (field == nullptr || !field->__isset.initial_default_value) { + generic_columns.push_back(name); + continue; + } + if (!this->col_name_to_block_idx_ref()->contains(name)) { + return Status::InternalError("Missing column: {} not found in block {}", name, + block->dump_structure()); + } + const auto position = this->col_name_to_block_idx_ref()->at(name); + ColumnPtr value; + RETURN_IF_ERROR(_build_owned_initial_default_column( + *field, block->get_by_position(position).type, rows, &value)); + // Iceberg metadata is authoritative for pre-add rows; a generic FE expression may be + // the Base64 carrier rather than the logical BINARY/FIXED/UUID value. + block->get_by_position(position).column = value->convert_to_full_column_if_const(); + } + return BaseReader::on_fill_missing_columns(block, rows, generic_columns); + } + // Called before reading a block: expand block for equality delete columns + detect row_id Status on_before_read_block(Block* block) override { RETURN_IF_ERROR(_expand_block_if_need(block)); @@ -270,6 +298,12 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { Status _expand_block_if_need(Block* block); Status _shrink_block_if_need(Block* block); + const schema::external::TStructField* _current_schema_root() const; + const schema::external::TField* _find_current_schema_field(const std::string& name) const; + const schema::external::TField* _find_current_schema_field(int32_t field_id) const; + Status _build_owned_initial_default_column(const schema::external::TField& field, + const DataTypePtr& type, size_t rows, + ColumnPtr* column) const; Status _register_missing_equality_delete_column(int32_t field_id, const std::string& name, const DataTypePtr& delete_key_type); Status _materialize_missing_equality_delete_column(Block* block, const std::string& name, @@ -664,6 +698,90 @@ Status IcebergReaderMixin::_expand_block_if_need(Block* block) { return Status::OK(); } +template +const schema::external::TStructField* IcebergReaderMixin::_current_schema_root() const { + const auto& scan_params = this->get_scan_params(); + if (!scan_params.__isset.history_schema_info || scan_params.history_schema_info.empty()) { + return nullptr; + } + const schema::external::TSchema* current_schema = &scan_params.history_schema_info.front(); + if (scan_params.__isset.current_schema_id) { + const auto schema_it = std::ranges::find_if( + scan_params.history_schema_info, [&](const schema::external::TSchema& schema) { + return schema.__isset.schema_id && + schema.schema_id == scan_params.current_schema_id; + }); + if (schema_it == scan_params.history_schema_info.end()) { + return nullptr; + } + current_schema = &*schema_it; + } + return current_schema->__isset.root_field ? ¤t_schema->root_field : nullptr; +} + +template +const schema::external::TField* IcebergReaderMixin::_find_current_schema_field( + const std::string& name) const { + const auto* root = _current_schema_root(); + if (root == nullptr) { + return nullptr; + } + const auto field = std::ranges::find_if(root->fields, [&](const auto& field_ptr) { + return field_ptr.__isset.field_ptr && field_ptr.field_ptr != nullptr && + field_ptr.field_ptr->__isset.name && field_ptr.field_ptr->name == name; + }); + return field == root->fields.end() ? nullptr : field->field_ptr.get(); +} + +template +const schema::external::TField* IcebergReaderMixin::_find_current_schema_field( + int32_t field_id) const { + const auto* root = _current_schema_root(); + if (root == nullptr) { + return nullptr; + } + const auto field = std::ranges::find_if(root->fields, [&](const auto& field_ptr) { + return field_ptr.__isset.field_ptr && field_ptr.field_ptr != nullptr && + field_ptr.field_ptr->__isset.id && field_ptr.field_ptr->id == field_id; + }); + return field == root->fields.end() ? nullptr : field->field_ptr.get(); +} + +template +Status IcebergReaderMixin::_build_owned_initial_default_column( + const schema::external::TField& field, const DataTypePtr& type, size_t rows, + ColumnPtr* column) const { + DORIS_CHECK(type != nullptr); + DORIS_CHECK(column != nullptr); + DORIS_CHECK(field.__isset.initial_default_value); + const auto nested_type = remove_nullable(type); + const bool is_base64 = + (field.__isset.initial_default_value_is_base64 && + field.initial_default_value_is_base64) || + (field.__isset.type && thrift_to_type(field.type.type) == TYPE_VARBINARY); + Field value; + if (is_base64) { + std::string decoded; + if (!base64_decode(field.initial_default_value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + field.name); + } + if (nested_type->get_primitive_type() == TYPE_VARBINARY) { + value = Field::create_field(StringView(decoded)); + } else { + DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); + value = Field::create_field(decoded); + } + // Variable-width Fields borrow decoded. Materialize before it leaves scope so every V1 + // missing-column and equality-delete boundary keeps UUID/FIXED payloads alive. + *column = type->create_column_const(rows, value); + return Status::OK(); + } + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string(field.initial_default_value, value)); + *column = type->create_column_const(rows, value); + return Status::OK(); +} + template Status IcebergReaderMixin::_register_missing_equality_delete_column( int32_t field_id, const std::string& name, const DataTypePtr& delete_key_type) { @@ -703,33 +821,15 @@ Status IcebergReaderMixin::_register_missing_equality_delete_column( "Missing Iceberg schema metadata for equality-delete field id {}", field_id); } - Field value; + ColumnPtr value_column; if (table_field->__isset.initial_default_value) { - const auto nested_type = remove_nullable(delete_key_type); - const bool default_is_base64 = (table_field->__isset.initial_default_value_is_base64 && - table_field->initial_default_value_is_base64) || - (table_field->__isset.type && - thrift_to_type(table_field->type.type) == TYPE_VARBINARY); - if (default_is_base64) { - std::string decoded_default; - if (!base64_decode(table_field->initial_default_value, &decoded_default)) { - return Status::InvalidArgument( - "Invalid Base64 Iceberg initial default for field {}", table_field->name); - } - if (nested_type->get_primitive_type() == TYPE_VARBINARY) { - value = Field::create_field(StringView(decoded_default)); - } else { - DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); - value = Field::create_field(decoded_default); - } - } else { - RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string( - table_field->initial_default_value, value)); - } + RETURN_IF_ERROR(_build_owned_initial_default_column(*table_field, delete_key_type, 1, + &value_column)); + } else { + value_column = delete_key_type->create_column_const(1, Field()); } - const bool inserted = _missing_equality_delete_values - .emplace(name, delete_key_type->create_column_const(1, value)) - .second; + const bool inserted = + _missing_equality_delete_values.emplace(name, std::move(value_column)).second; DORIS_CHECK(inserted); this->register_synthesized_column_handler( name, [this, name](Block* block, size_t rows) -> Status { @@ -996,6 +1096,9 @@ Status IcebergReaderMixin::_gen_position_delete_file_range( template Status IcebergReaderMixin::read_deletion_vector( const std::string& data_file_path, const TIcebergDeleteFileDesc& delete_file_desc) { + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_iceberg_deletion_vector_descriptor(delete_file_desc, bytes_read)); + Status create_status = Status::OK(); SCOPED_TIMER(_iceberg_profile.delete_files_read_time); bool decoded_cache_hit = false; diff --git a/be/src/format/table/iceberg_scan_semantics.h b/be/src/format/table/iceberg_scan_semantics.h new file mode 100644 index 00000000000000..f579f063b76327 --- /dev/null +++ b/be/src/format/table/iceberg_scan_semantics.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { + +inline constexpr int32_t ICEBERG_SCAN_SEMANTICS_VERSION_1 = 1; + +inline bool supports_iceberg_scan_semantics_v1(const TFileScanRangeParams* params) { + // Old FE plans can carry IDs and encoded defaults too, so only this explicit version marker + // may opt a new BE into result-changing semantics during a rolling upgrade. + return params != nullptr && params->__isset.iceberg_scan_semantics_version && + params->iceberg_scan_semantics_version >= ICEBERG_SCAN_SEMANTICS_VERSION_1; +} + +} // namespace doris diff --git a/be/src/format/table/paimon_reader.cpp b/be/src/format/table/paimon_reader.cpp index 4953078c32dc02..a73d65f8336cc1 100644 --- a/be/src/format/table/paimon_reader.cpp +++ b/be/src/format/table/paimon_reader.cpp @@ -42,26 +42,41 @@ std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc deletion_file.length); } +Status validate_paimon_deletion_vector_descriptor(const TPaimonDeletionFileDesc& deletion_file, + size_t& bytes_read) { + if (!deletion_file.__isset.path || !deletion_file.__isset.offset || + !deletion_file.__isset.length) { + return Status::DataQualityError( + "Paimon deletion file descriptor misses path/offset/length"); + } + return validate_paimon_deletion_vector_read_range(deletion_file.offset, deletion_file.length, + bytes_read); +} + Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, DeletionVector* deletion_vector) { if (deletion_vector == nullptr) { - return Status::InvalidArgument("deletion_vector must not be null"); + return Status::InvalidArgument("deletion vector output must not be null"); + } + if (buf == nullptr) { + return Status::DataQualityError("Paimon deletion vector blob is null"); } if (buffer_size < 8) [[unlikely]] { return Status::DataQualityError("Deletion vector file size too small: {}", buffer_size); } const uint32_t actual_length = BigEndian::Load32(buf); - if (actual_length + 4 != buffer_size) [[unlikely]] { - return Status::RuntimeError( - "DeletionVector deserialize error: length not match, " - "actual length: {}, expect length: {}", - actual_length, buffer_size - 4); + if (static_cast(actual_length) + 4 != buffer_size) [[unlikely]] { + return Status::DataQualityError( + "Paimon deletion vector length mismatch, expected: {}, actual: {}", + static_cast(actual_length) + 4, buffer_size); } if (memcmp(buf + sizeof(actual_length), PAIMON_BITMAP_MAGIC, 4) != 0) [[unlikely]] { - return Status::RuntimeError("DeletionVector deserialize error: invalid magic number {}", - BigEndian::Load32(buf + sizeof(actual_length))); + return Status::DataQualityError( + "Paimon deletion vector magic number mismatch, expected: {}, actual: {}", + BigEndian::Load32(PAIMON_BITMAP_MAGIC), + BigEndian::Load32(buf + sizeof(actual_length))); } roaring::Roaring roaring_bitmap; @@ -158,6 +173,8 @@ Status PaimonOrcReader::_init_deletion_vector() { set_push_down_agg_type(TPushAggOp::NONE); } const auto& deletion_file = table_desc.deletion_file; + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read)); Status create_status = Status::OK(); @@ -172,7 +189,7 @@ Status PaimonOrcReader::_init_deletion_vector() { delete_range.__set_fs_name(get_scan_range().fs_name); delete_range.path = deletion_file.path; delete_range.start_offset = deletion_file.offset; - delete_range.size = deletion_file.length + 4; + delete_range.size = static_cast(bytes_read); delete_range.file_size = -1; DeletionVectorReader dv_reader(get_state(), get_profile(), get_scan_params(), @@ -182,7 +199,6 @@ Status PaimonOrcReader::_init_deletion_vector() { return nullptr; } - size_t bytes_read = deletion_file.length + 4; std::vector buffer(bytes_read); create_status = dv_reader.read_at(deletion_file.offset, {buffer.data(), bytes_read}); @@ -264,6 +280,8 @@ Status PaimonParquetReader::_init_deletion_vector() { set_push_down_agg_type(TPushAggOp::NONE); } const auto& deletion_file = table_desc.deletion_file; + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read)); Status create_status = Status::OK(); @@ -278,7 +296,7 @@ Status PaimonParquetReader::_init_deletion_vector() { delete_range.__set_fs_name(get_scan_range().fs_name); delete_range.path = deletion_file.path; delete_range.start_offset = deletion_file.offset; - delete_range.size = deletion_file.length + 4; + delete_range.size = static_cast(bytes_read); delete_range.file_size = -1; DeletionVectorReader dv_reader(get_state(), get_profile(), get_scan_params(), @@ -288,7 +306,6 @@ Status PaimonParquetReader::_init_deletion_vector() { return nullptr; } - size_t bytes_read = deletion_file.length + 4; std::vector buffer(bytes_read); create_status = dv_reader.read_at(deletion_file.offset, {buffer.data(), bytes_read}); diff --git a/be/src/format/table/paimon_reader.h b/be/src/format/table/paimon_reader.h index 0f11825ce89d15..746ee1adec1e3a 100644 --- a/be/src/format/table/paimon_reader.h +++ b/be/src/format/table/paimon_reader.h @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -35,6 +36,9 @@ class ShardedKVCache; std::string build_paimon_deletion_vector_cache_key(const TPaimonDeletionFileDesc& deletion_file); +Status validate_paimon_deletion_vector_descriptor(const TPaimonDeletionFileDesc& deletion_file, + size_t& bytes_read); + Status decode_paimon_deletion_vector_buffer(const char* buf, size_t buffer_size, DeletionVector* deletion_vector); diff --git a/be/src/format/table/table_schema_change_helper.cpp b/be/src/format/table/table_schema_change_helper.cpp index 3d4cb6b5a6c99c..57fed2d11db066 100644 --- a/be/src/format/table/table_schema_change_helper.cpp +++ b/be/src/format/table/table_schema_change_helper.cpp @@ -55,6 +55,39 @@ std::map build_lowercase_orc_field_name_idx_map(const orc:: return file_column_name_idx_map; } +bool orc_subtree_has_field_id(const orc::Type* type, const std::string& attribute) { + if (type->hasAttributeKey(attribute)) { + return true; + } + for (uint64_t idx = 0; idx < type->getSubtypeCount(); ++idx) { + if (orc_subtree_has_field_id(type->getSubtype(idx), attribute)) { + return true; + } + } + return false; +} + +bool orc_children_all_have_field_ids(const orc::Type* type, const std::string& attribute) { + for (uint64_t idx = 0; idx < type->getSubtypeCount(); ++idx) { + if (!type->getSubtype(idx)->hasAttributeKey(attribute)) { + return false; + } + } + return true; +} + +std::optional initial_default_value( + const schema::external::TField& field) { + if (!field.__isset.initial_default_value) { + return std::nullopt; + } + return TableSchemaChangeHelper::InitialDefaultValue { + .value = field.initial_default_value, + .is_base64 = field.__isset.initial_default_value_is_base64 && + field.initial_default_value_is_base64, + }; +} + bool find_file_field_idx_by_name_mapping( const schema::external::TField& table_field, const std::map& file_column_name_idx_map, size_t* file_column_idx) { @@ -73,11 +106,103 @@ bool find_file_field_idx_by_name_mapping( return true; } } + if (table_field.__isset.name_mapping_is_authoritative && + table_field.name_mapping_is_authoritative) { + // Only a compatible FE can make the mapping authoritative; older FE plans must retain + // their legacy current-name fallback throughout a rolling BE upgrade. + return false; + } } return table_field.__isset.name && try_match(table_field.name); } +bool table_subtree_contains_field_id(const schema::external::TField& field, int32_t field_id) { + if (field.id == field_id) { + return true; + } + if (!field.__isset.nestedField) { + return false; + } + switch (field.type.type) { + case TPrimitiveType::STRUCT: + if (field.nestedField.__isset.struct_field) { + for (const auto& child : field.nestedField.struct_field.fields) { + if (child.field_ptr != nullptr && + table_subtree_contains_field_id(*child.field_ptr, field_id)) { + return true; + } + } + } + break; + case TPrimitiveType::ARRAY: + if (field.nestedField.__isset.array_field && + field.nestedField.array_field.__isset.item_field && + field.nestedField.array_field.item_field.field_ptr != nullptr) { + return table_subtree_contains_field_id( + *field.nestedField.array_field.item_field.field_ptr, field_id); + } + break; + case TPrimitiveType::MAP: + if (field.nestedField.__isset.map_field) { + const auto& map = field.nestedField.map_field; + if (map.__isset.key_field && map.key_field.field_ptr != nullptr && + table_subtree_contains_field_id(*map.key_field.field_ptr, field_id)) { + return true; + } + if (map.__isset.value_field && map.value_field.field_ptr != nullptr) { + return table_subtree_contains_field_id(*map.value_field.field_ptr, field_id); + } + } + break; + default: + break; + } + return false; +} + +bool has_shared_descendant_field_id(const schema::external::TField& table_field, + const FieldSchema& file_field) { + for (const auto& file_child : file_field.children) { + if ((file_child.field_id != -1 && + table_subtree_contains_field_id(table_field, file_child.field_id)) || + has_shared_descendant_field_id(table_field, file_child)) { + return true; + } + } + return false; +} + +std::optional find_unique_idless_wrapper( + const schema::external::TField& table_field, + const std::vector& parquet_fields_schema) { + std::optional match; + for (size_t idx = 0; idx < parquet_fields_schema.size(); ++idx) { + const auto& candidate = parquet_fields_schema[idx]; + if (candidate.field_id != -1 || candidate.children.empty() || + !has_shared_descendant_field_id(table_field, candidate)) { + continue; + } + if (match.has_value()) { + return std::nullopt; + } + match = idx; + } + return match; +} + +bool parquet_subtree_has_field_id(const FieldSchema& field) { + if (field.field_id != -1) { + return true; + } + return std::ranges::any_of(field.children, parquet_subtree_has_field_id); +} + +bool parquet_fields_all_have_field_ids(const std::vector& fields) { + return std::ranges::all_of(fields, + [](const FieldSchema& field) { return field.field_id != -1; }); +} + } // namespace Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name( @@ -534,7 +659,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id( exist_field_id)); struct_node->add_children(table_column_name, file_field.name, field_node); } else { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children(table_column_name, + initial_default_value(*table_field.field_ptr)); } } node = struct_node; @@ -552,21 +678,29 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const schema::external::TStructField& table_schema, const FieldDescriptor& parquet_field_desc, std::shared_ptr& node) { + return by_parquet_field_id_with_name_mapping(table_schema, parquet_field_desc, node, false); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, + const FieldDescriptor& parquet_field_desc, + std::shared_ptr& node, bool use_current_iceberg_semantics) { auto struct_node = std::make_shared(); const auto& parquet_fields_schema = parquet_field_desc.get_fields_schema(); std::map file_column_id_idx_map; - bool all_have_field_id = true; + const bool use_field_id = + use_current_iceberg_semantics + ? std::ranges::any_of(parquet_fields_schema, parquet_subtree_has_field_id) + : parquet_fields_all_have_field_ids(parquet_fields_schema); for (size_t idx = 0; idx < parquet_fields_schema.size(); idx++) { - if (parquet_fields_schema[idx].field_id == -1) { - all_have_field_id = false; - break; + if (parquet_fields_schema[idx].field_id != -1) { + file_column_id_idx_map.emplace(parquet_fields_schema[idx].field_id, idx); } - file_column_id_idx_map.emplace(parquet_fields_schema[idx].field_id, idx); } std::map file_column_name_idx_map; - if (!all_have_field_id) { + if (!use_field_id) { file_column_name_idx_map = build_lowercase_field_name_idx_map(parquet_fields_schema); } @@ -574,11 +708,21 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& table_column_name = table_field.field_ptr->name; size_t file_column_idx = 0; bool matched = false; - if (all_have_field_id) { + if (use_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; matched = true; + } else if (use_current_iceberg_semantics) { + auto wrapper = + find_unique_idless_wrapper(*table_field.field_ptr, parquet_fields_schema); + if (wrapper.has_value()) { + // Parquet may retain a selected struct without its own ID; a unique + // descendant-ID match is authoritative even when Iceberg name mapping + // intentionally has no alias. + file_column_idx = *wrapper; + matched = true; + } } } else { matched = find_file_field_idx_by_name_mapping( @@ -586,13 +730,17 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children( + table_column_name, use_current_iceberg_semantics + ? initial_default_value(*table_field.field_ptr) + : std::nullopt); continue; } std::shared_ptr field_node = nullptr; RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( - *table_field.field_ptr, parquet_fields_schema[file_column_idx], field_node)); + *table_field.field_ptr, parquet_fields_schema[file_column_idx], field_node, + use_field_id, use_current_iceberg_semantics)); struct_node->add_children(table_column_name, parquet_fields_schema[file_column_idx].name, field_node); } @@ -604,6 +752,13 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( const schema::external::TField& table_schema, const FieldSchema& parquet_field, std::shared_ptr& node) { + return by_parquet_field_id_with_name_mapping(table_schema, parquet_field, node, false, false); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + const schema::external::TField& table_schema, const FieldSchema& parquet_field, + std::shared_ptr& node, bool use_field_id, + bool use_current_iceberg_semantics) { switch (table_schema.type.type) { case TPrimitiveType::MAP: { if (parquet_field.data_type->get_primitive_type() != TYPE_MAP) [[unlikely]] { @@ -623,10 +778,11 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( *table_schema.nestedField.map_field.key_field.field_ptr, parquet_field.children[0], - key_node)); + key_node, use_field_id, use_current_iceberg_semantics)); RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( *table_schema.nestedField.map_field.value_field.field_ptr, - parquet_field.children[1], value_node)); + parquet_field.children[1], value_node, use_field_id, + use_current_iceberg_semantics)); node = std::make_shared(key_node, value_node); break; @@ -645,7 +801,8 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam std::shared_ptr element_node = nullptr; RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( *table_schema.nestedField.array_field.item_field.field_ptr, - parquet_field.children[0], element_node)); + parquet_field.children[0], element_node, use_field_id, + use_current_iceberg_semantics)); node = std::make_shared(element_node); break; @@ -659,18 +816,18 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam auto struct_node = std::make_shared(); + if (!use_current_iceberg_semantics) { + use_field_id = parquet_fields_all_have_field_ids(parquet_field.children); + } std::map file_column_id_idx_map; - bool all_have_field_id = true; for (size_t idx = 0; idx < parquet_field.children.size(); idx++) { - if (parquet_field.children[idx].field_id == -1) { - all_have_field_id = false; - break; + if (parquet_field.children[idx].field_id != -1) { + file_column_id_idx_map.emplace(parquet_field.children[idx].field_id, idx); } - file_column_id_idx_map.emplace(parquet_field.children[idx].field_id, idx); } std::map file_column_name_idx_map; - if (!all_have_field_id) { + if (!use_field_id) { file_column_name_idx_map = build_lowercase_field_name_idx_map(parquet_field.children); } @@ -678,11 +835,19 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam const auto& table_column_name = table_field.field_ptr->name; size_t file_column_idx = 0; bool matched = false; - if (all_have_field_id) { + if (use_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_column_idx = id_it->second; matched = true; + } else if (use_current_iceberg_semantics) { + auto wrapper = find_unique_idless_wrapper(*table_field.field_ptr, + parquet_field.children); + if (wrapper.has_value()) { + // Apply the same Parquet wrapper invariant at every struct recursion depth. + file_column_idx = *wrapper; + matched = true; + } } } else { matched = find_file_field_idx_by_name_mapping( @@ -690,14 +855,18 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_nam } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children( + table_column_name, use_current_iceberg_semantics + ? initial_default_value(*table_field.field_ptr) + : std::nullopt); continue; } const auto& file_field = parquet_field.children.at(file_column_idx); std::shared_ptr field_node = nullptr; - RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping(*table_field.field_ptr, - file_field, field_node)); + RETURN_IF_ERROR(by_parquet_field_id_with_name_mapping( + *table_field.field_ptr, file_field, field_node, use_field_id, + use_current_iceberg_semantics)); struct_node->add_children(table_column_name, file_field.name, field_node); } node = struct_node; @@ -822,22 +991,40 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const schema::external::TStructField& table_schema, const orc::Type* orc_root, const std::string& field_id_attribute_key, std::shared_ptr& node) { + return by_orc_field_id_with_name_mapping(table_schema, orc_root, field_id_attribute_key, node, + false); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_current_iceberg_semantics) { + const bool use_field_id = + use_current_iceberg_semantics + ? orc_subtree_has_field_id(orc_root, field_id_attribute_key) + : orc_children_all_have_field_ids(orc_root, field_id_attribute_key); + return by_orc_field_id_with_name_mapping(table_schema, orc_root, field_id_attribute_key, node, + use_field_id, use_current_iceberg_semantics); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id, + bool use_current_iceberg_semantics) { auto struct_node = std::make_shared(); std::map file_column_id_idx_map; - bool all_have_field_id = true; for (size_t idx = 0; idx < orc_root->getSubtypeCount(); idx++) { - if (!orc_root->getSubtype(idx)->hasAttributeKey(field_id_attribute_key)) { - all_have_field_id = false; - break; + if (orc_root->getSubtype(idx)->hasAttributeKey(field_id_attribute_key)) { + auto field_id = + std::stoi(orc_root->getSubtype(idx)->getAttributeValue(field_id_attribute_key)); + file_column_id_idx_map.emplace(field_id, idx); } - auto field_id = - std::stoi(orc_root->getSubtype(idx)->getAttributeValue(field_id_attribute_key)); - file_column_id_idx_map.emplace(field_id, idx); } std::map file_column_name_idx_map; - if (!all_have_field_id) { + if (!use_field_id) { file_column_name_idx_map = build_lowercase_orc_field_name_idx_map(orc_root); } @@ -845,7 +1032,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const auto& table_column_name = table_field.field_ptr->name; size_t file_field_idx = 0; bool matched = false; - if (all_have_field_id) { + if (use_field_id) { auto id_it = file_column_id_idx_map.find(table_field.field_ptr->id); if (id_it != file_column_id_idx_map.end()) { file_field_idx = id_it->second; @@ -857,14 +1044,18 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma } if (!matched) { - struct_node->add_not_exist_children(table_column_name); + struct_node->add_not_exist_children( + table_column_name, use_current_iceberg_semantics + ? initial_default_value(*table_field.field_ptr) + : std::nullopt); continue; } const auto& file_field = orc_root->getSubtype(file_field_idx); std::shared_ptr field_node = nullptr; - RETURN_IF_ERROR(by_orc_field_id_with_name_mapping(*table_field.field_ptr, file_field, - field_id_attribute_key, field_node)); + RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( + *table_field.field_ptr, file_field, field_id_attribute_key, field_node, + use_field_id, use_current_iceberg_semantics)); struct_node->add_children(table_column_name, orc_root->getFieldName(file_field_idx), field_node); } @@ -876,6 +1067,15 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma const schema::external::TField& table_schema, const orc::Type* orc_root, const std::string& field_id_attribute_key, std::shared_ptr& node) { + return by_orc_field_id_with_name_mapping(table_schema, orc_root, field_id_attribute_key, node, + false, false); +} + +Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + const schema::external::TField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id, + bool use_current_iceberg_semantics) { switch (table_schema.type.type) { case TPrimitiveType::MAP: { if (orc_root->getKind() != orc::TypeKind::MAP) [[unlikely]] { @@ -895,10 +1095,10 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( *table_schema.nestedField.map_field.key_field.field_ptr, orc_root->getSubtype(0), - field_id_attribute_key, key_node)); + field_id_attribute_key, key_node, use_field_id, use_current_iceberg_semantics)); RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( *table_schema.nestedField.map_field.value_field.field_ptr, orc_root->getSubtype(1), - field_id_attribute_key, value_node)); + field_id_attribute_key, value_node, use_field_id, use_current_iceberg_semantics)); node = std::make_shared(key_node, value_node); break; @@ -917,7 +1117,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma std::shared_ptr element_node = nullptr; RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( *table_schema.nestedField.array_field.item_field.field_ptr, orc_root->getSubtype(0), - field_id_attribute_key, element_node)); + field_id_attribute_key, element_node, use_field_id, use_current_iceberg_semantics)); node = std::make_shared(element_node); break; @@ -928,8 +1128,12 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_ma } MOCK_REMOVE(DCHECK(table_schema.__isset.nestedField)); MOCK_REMOVE(DCHECK(table_schema.nestedField.__isset.struct_field)); - RETURN_IF_ERROR(by_orc_field_id_with_name_mapping(table_schema.nestedField.struct_field, - orc_root, field_id_attribute_key, node)); + if (!use_current_iceberg_semantics) { + use_field_id = orc_children_all_have_field_ids(orc_root, field_id_attribute_key); + } + RETURN_IF_ERROR(by_orc_field_id_with_name_mapping( + table_schema.nestedField.struct_field, orc_root, field_id_attribute_key, node, + use_field_id, use_current_iceberg_semantics)); break; } default: { diff --git a/be/src/format/table/table_schema_change_helper.h b/be/src/format/table/table_schema_change_helper.h index 2d54b3216cac89..b5d377c23ab508 100644 --- a/be/src/format/table/table_schema_change_helper.h +++ b/be/src/format/table/table_schema_change_helper.h @@ -19,6 +19,7 @@ #include #include +#include #include #include "common/status.h" @@ -41,6 +42,10 @@ namespace doris { class TableSchemaChangeHelper { public: + struct InitialDefaultValue { + std::string value; + bool is_base64 = false; + }; ~TableSchemaChangeHelper() = default; class Node { @@ -67,6 +72,11 @@ class TableSchemaChangeHelper { "children_column_exists should not be called on base TableInfoNode"); } + virtual std::optional children_initial_default_value( + std::string) const { + return std::nullopt; + } + virtual std::shared_ptr get_element_node() const { throw std::logic_error("get_element_node should not be called on base TableInfoNode"); } @@ -78,7 +88,9 @@ class TableSchemaChangeHelper { throw std::logic_error("get_value_node should not be called on base TableInfoNode"); } - virtual void add_not_exist_children(std::string table_column_name) { + virtual void add_not_exist_children( + std::string table_column_name, + std::optional initial_default = std::nullopt) { throw std::logic_error( "add_not_exist_children should not be called on base TableInfoNode"); }; @@ -131,6 +143,7 @@ class TableSchemaChangeHelper { const std::shared_ptr node; const std::string column_name; const bool exists; + const std::optional initial_default; }; // table column name -> { node, file_column_name, exists_in_file} @@ -167,14 +180,23 @@ class TableSchemaChangeHelper { return children.at(table_column_name).exists; } - void add_not_exist_children(std::string table_column_name) override { - children.emplace(table_column_name, StructChild {nullptr, "", false}); + std::optional children_initial_default_value( + std::string table_column_name) const override { + DCHECK(children.contains(table_column_name)); + return children.at(table_column_name).initial_default; + } + + void add_not_exist_children( + std::string table_column_name, + std::optional initial_default = std::nullopt) override { + children.emplace(table_column_name, + StructChild {nullptr, "", false, std::move(initial_default)}); } void add_children(std::string table_column_name, std::string file_column_name, std::shared_ptr children_node) override { children.emplace(table_column_name, - StructChild {children_node, file_column_name, true}); + StructChild {children_node, file_column_name, true, std::nullopt}); } const std::map& get_children() const { return children; } @@ -325,11 +347,22 @@ class TableSchemaChangeHelper { const FieldDescriptor& parquet_field_desc, std::shared_ptr& node); + static Status by_parquet_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, + const FieldDescriptor& parquet_field_desc, + std::shared_ptr& node, + bool use_current_iceberg_semantics); + // for iceberg parquet static Status by_parquet_field_id_with_name_mapping( const schema::external::TField& table_schema, const FieldSchema& parquet_field, std::shared_ptr& node); + static Status by_parquet_field_id_with_name_mapping( + const schema::external::TField& table_schema, const FieldSchema& parquet_field, + std::shared_ptr& node, bool use_field_id, + bool use_current_iceberg_semantics); + // for iceberg orc : Use the field id in the `table schema` and the orc file to match columns. static Status by_orc_field_id(const schema::external::TStructField& table_schema, const orc::Type* orc_root, @@ -351,11 +384,29 @@ class TableSchemaChangeHelper { const std::string& field_id_attribute_key, std::shared_ptr& node); + static Status by_orc_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, + bool use_current_iceberg_semantics); + + static Status by_orc_field_id_with_name_mapping( + const schema::external::TStructField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id, + bool use_current_iceberg_semantics); + // for iceberg orc static Status by_orc_field_id_with_name_mapping( const schema::external::TField& table_schema, const orc::Type* orc_root, const std::string& field_id_attribute_key, std::shared_ptr& node); + + static Status by_orc_field_id_with_name_mapping( + const schema::external::TField& table_schema, const orc::Type* orc_root, + const std::string& field_id_attribute_key, + std::shared_ptr& node, bool use_field_id, + bool use_current_iceberg_semantics); }; }; diff --git a/be/src/format/transformer/vorc_transformer.cpp b/be/src/format/transformer/vorc_transformer.cpp index 7dfa9fb4c64d3b..7ddab79dc1c6b8 100644 --- a/be/src/format/transformer/vorc_transformer.cpp +++ b/be/src/format/transformer/vorc_transformer.cpp @@ -385,12 +385,19 @@ Status VOrcTransformer::collect_file_statistics_after_close(TIcebergColumnStats* const iceberg::StructType& root_struct = _iceberg_schema->root_struct(); const auto& nested_fields = root_struct.fields(); + const orc::Type& orc_root_type = reader->getType(); for (uint32_t i = 0; i < nested_fields.size(); i++) { - uint32_t orc_col_id = i + 1; // skip root struct - if (orc_col_id >= file_stats->getNumberOfColumns()) { + if (i >= orc_root_type.getSubtypeCount()) { + continue; + } + // ORC IDs are depth-first, so top-level fields after a complex field are not i + 1. + const uint64_t raw_orc_col_id = orc_root_type.getSubtype(i)->getColumnId(); + if (raw_orc_col_id >= file_stats->getNumberOfColumns()) { continue; } + // The uint32_t column-count check above makes narrowing to the ORC API width safe. + const uint32_t orc_col_id = static_cast(raw_orc_col_id); const orc::ColumnStatistics* col_stats = file_stats->getColumnStatistics(orc_col_id); if (col_stats == nullptr) { continue; diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index 377a26ce530db8..e417617cf5a64f 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -108,6 +108,107 @@ instructions as well; this file adds format-v2-specific review expectations. - For JNI readers, review local/global reference lifetime, exception propagation, type conversion, thread attachment assumptions, and cleanup on partial initialization. +### Parquet Native Decode Kernel + +- Keep new production integration under `be/src/format_v2/parquet/`. Doris v1 is the behavior and + performance baseline, but v2 owns an independent page/encoding reader and must not call the v1 + `ParquetColumnReader`. Do not modify `be/src/format/parquet/` for a v2 decoder change. Reimplement + the required behavior under the v2 tree and keep v1 unchanged so differential correctness and + performance results remain meaningful. +- Keep the native decode boundary independent of both Arrow descriptors/builders and table-schema + objects. A Column Chunk schema contract should contain only immutable physical type, fixed width, + and Dremel-level thresholds. Review constructor arguments and stored references for ownership and + lifetime; metadata owned by a temporary schema adapter must not escape into a persistent reader. +- Treat selection positions as logical Row Group rows, including null rows. Selection indices must + be sorted, unique, and bounded by the batch's logical row count. Dense identity, empty selection, + and fragmented selection must have explicit representations and tests; do not silently mix row + ordinals with non-null value ordinals or dictionary IDs. +- Verify the three decode counts separately: logical rows consumed, encoded non-null payload values + consumed, and output values materialized. Null and filtered-null runs consume no payload; + selected and filtered non-null runs both consume payload. Every page transition, skip, error, and + end-of-batch path must leave all three cursors aligned for the next call. +- A flat scalar fast path may run only when `max_repetition_level == 0`. Repeated leaves require a + definition/repetition-level plan that identifies parent-row boundaries, empty and null + collections, null ancestors, and rows spanning pages. Choose one physical leaf as the parent + shape owner; sibling leaf streams advance over the same parent-row range and validate their + payload counts instead of independently redefining the parent shape. +- The old Arrow value-reader hierarchy (`ParquetLeafBatch`, scalar/list/map/struct readers, and the + nested load/build/consume protocol) has been removed. Do not reintroduce an intermediate decoded + batch or a stateful load-before-build phase. Ordinary predicate/output scans construct + `NativeColumnReader`, consume compressed page data through the native decoder, and append directly + into the final Doris column. +- Keep logical schema changes distinct from physical decoding. The native reader materializes the + projected file type only; `ColumnMapper` and `TableReader` own every file-to-table cast. A type + mismatch at the native reader boundary is an invariant violation, not a reason to create a + reader-local conversion path or expose a decoder-owned value batch. +- `CountColumnReader` uses the v2 native `LevelReader`; it selects one representative leaf (the key + for MAP) and advances only definition/repetition levels. It exposes no value API and must not be + reused as a scan reader or expanded into a fallback path. +- Build ARRAY/MAP/STRUCT parent boundaries, offsets, nulls, and child payload spans in one traversal + of the owning leaf's levels. For example, `[[1, 2], NULL, []]` must yield entry counts + `[2, 0, 0]` and parent nulls `[0, 1, 0]` without rescanning that leaf. MAP key levels own entry + existence; value levels validate against that shape. STRUCT siblings validate parent-row + alignment. If every projected STRUCT child is missing, consume only a retained physical leaf's + levels and never materialize its payload into a temporary Doris column. +- Do not size level or selection scratch from a 16-bit batch-row assumption. A repeated parent row + can contain more level entries than the requested parent-row batch. Split large runs without + changing alternation or row-boundary semantics, and check overflow before narrowing counts. +- Decoder dispatch must reject an incompatible physical type, encoding, or type length explicitly. + Review PLAIN, dictionary/RLE, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, BYTE_STREAM_SPLIT, BOOLEAN RLE, and level RLE/bit-packed paths for identical + selection and malformed-input behavior. Page V1 and V2 must feed the same decoder contract after + their different level/decompression layouts are parsed. +- Keep every index coordinate domain explicit: table-local column ID, physical leaf-column ID, + Row Group ID, data-page ordinal, OffsetIndex row ordinal, logical batch-row ordinal, non-null + payload ordinal, and dictionary-entry ID are different types of identity. Data-page ordinals must + exclude dictionary pages consistently. Dictionary-entry bitmaps are local to one Column Chunk + dictionary and cannot be reused after a Row Group, dictionary, or encoding transition. +- Review index composition, not only each index in isolation. Row Group statistics, dictionary, + Bloom, ColumnIndex/OffsetIndex, page cache registration, page skip plans, SelectionVector, and + lazy column cursors must describe the same surviving logical rows. Missing or unusable optional + indexes retain candidates; structurally inconsistent indexes or out-of-range IDs return an + explicit corruption error. A mixed dictionary/plain Column Chunk must leave dictionary-ID + filtering before any cursor is consumed. +- Decoder owns encoded-stream parsing and cursor movement only. `DataTypeSerDe` owns Parquet + physical/logical interpretation and writes directly into Doris columns. Dictionary pages are + materialized once through the same SerDe and data pages expose only validated dictionary indices. + Decimal and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, + precision, and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. +- Do not add an Arrow runtime fallback. Once an ordinary v2 scan selects its native Parquet reader, + unsupported physical types, encodings, page layouts, or malformed inputs return an explicit + status. Arrow may be used by metadata planning and as a test oracle; no Arrow array, builder, + RecordReader, or metadata lifetime belongs in data-page value or level materialization. +- Reuse decoder, SerDe, null-map, selection-range, binary-value, level, and builder scratch across + batches. String-like decoders should gather selected `StringRef` values and append once per batch, + rather than allocate or grow the destination once per run. Scratch capacity may grow to a bounded + high-water mark but must be reset logically between pages, Row Groups, files, and errors. +- Adaptive batch sizing must measure completed Doris output rows/bytes and must not recreate native + readers, builders, or scratch when only the row cap changes. Compare the v1 and v2 lifecycle: + probe batches must not turn persistent setup into a per-batch cost or amplify highly fragmented + selection work. +- Footer and metadata caching must key on stable file identity and cache the serialized footer plus + parsed native metadata at the same lifecycle as v1. Never reuse metadata when path, file size, + modification/version identity, encryption state, or schema-affecting options differ. Cache misses + and uncacheable identities must remain correct without a fallback to stale entries. +- Page-cache behavior must match v1 for cache key, stable file identity, registered byte range, + compressed/decompressed entry kind, checksum/decompression ownership, subrange coverage, + invalidation, admission, and fallback I/O. Any intentional difference needs benchmark and memory + evidence showing it is no worse for v1 workloads. Cache lookup must never alter page ordinal, + decoder, level, or dictionary cursor state. +- Apply v1's MergeRange decision to the native data-page reader, after metadata/dictionary probes + finish. Predicate and lazy readers for one Row Group must share one ordered-range wrapper, and + native per-column prefetch must be disabled while that wrapper is active. Never allocate one + MergeRange buffer per leaf; wide complex projections would multiply its bounded scratch memory. +- Preserve observability inside aggregate counters. `TotalBatches` must be decomposable into probe, + dense, selected, empty, page-crossing, and nested/fragmented work where relevant; decode, level, + selection, conversion, allocation, and materialization time must remain attributable without + adding per-row timer overhead. +- Profile index attempts, successes, conservative fallbacks, and corrupt rejections separately for + statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skipping. Footer/page/file/ + condition-cache counters must expose requests, hits, misses, writes/admissions, bytes, wait/I/O + time, and bypass reasons with semantics aligned to v1. A lower total timer without its internal + work counters is not sufficient observability. + ## Detailed FileReader Review Guides - Before reviewing any FileReader implementation, index, predicate path, cache, or virtual column, @@ -174,6 +275,15 @@ instructions as well; this file adds format-v2-specific review expectations. malformed input. - For bug fixes, require a test that fails for the original reachable path and validates the result, row count, or explicit error after the fix. +- For native Parquet work, require a matrix across physical/logical types, every supported encoding, + Page V1/V2, required/optional/repeated levels, dictionary fallback, page and batch boundaries, + dense/empty/fragmented selections, null placement, malformed/truncated input, and files from + representative external writers. Include focused cursor-invariant tests and end-to-end nested + reconstruction tests; a scalar happy-path benchmark is not sufficient coverage. +- Performance-sensitive decoder changes need a reproducible comparison against v1 and a relevant + external baseline such as DuckDB. Report data shape, encoding, selectivity, null/cardinality + distribution, compression, storage path, batch policy, warm/cold cache state, CPU time, rows/s, + bytes/s, allocation behavior, and Profile counter deltas. ## Review Output diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index ac510caaf07ac4..d504305fad4014 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -248,6 +248,9 @@ struct ColumnDefinition { // Historical or external names for the same logical field. Table formats such as Iceberg can // use this to resolve partition path keys after column rename. std::vector name_mapping {}; + // Distinguishes no table-level mapping from an explicit empty field mapping. The latter must + // not fall back to the current name when matching fields in legacy files. + bool has_name_mapping = false; DataTypePtr type; // Semantic nested children for this schema node. // @@ -255,6 +258,9 @@ struct ColumnDefinition { // FileReader::get_schema() also expose semantic children, not physical reader wrappers. For // example, MAP children are key/value and ARRAY children contain only the element field. std::vector children {}; + // Full table-schema identity subtree before access-path pruning. ID-less physical complex + // wrappers must be discovered from this view without adding unrequested children to output. + std::vector identity_children {}; // Expression used to materialize missing/default/generated values when the column is not read // directly from the file. VExprContextSPtr default_expr = nullptr; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 6a0daf903f01d4..ed0dcda9739d6b 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -51,11 +52,77 @@ #include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" +#include "util/url_coding.h" namespace doris::format { namespace { +Status build_initial_default_column(const ColumnDefinition& column, ColumnPtr* value) { + DORIS_CHECK(value != nullptr); + *value = nullptr; + if (!column.initial_default_value.has_value()) { + return Status::OK(); + } + const auto nested_type = remove_nullable(column.type); + Field parsed; + if (column.initial_default_value_is_base64 || + nested_type->get_primitive_type() == TYPE_VARBINARY) { + std::string decoded; + if (!base64_decode(*column.initial_default_value, &decoded)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + column.name); + } + parsed = nested_type->get_primitive_type() == TYPE_VARBINARY + ? Field::create_field(StringView(decoded)) + : Field::create_field(decoded); + // Variable-width Fields borrow their input. Materialize while decoded is alive so the + // resulting column owns the payload before it crosses a mapping/literal boundary. + *value = column.type->create_column_const(1, parsed); + return Status::OK(); + } else { + RETURN_IF_ERROR( + nested_type->get_serde()->from_fe_string(*column.initial_default_value, parsed)); + } + *value = column.type->create_column_const(1, parsed); + return Status::OK(); +} + +Status build_initial_default_literal(const ColumnDefinition& column, VExprContextSPtr* literal) { + DORIS_CHECK(literal != nullptr); + ColumnPtr owned_value; + RETURN_IF_ERROR(build_initial_default_column(column, &owned_value)); + DORIS_CHECK(static_cast(owned_value)); + Field value; + owned_value->get(0, value); + // VLiteral copies the borrowed Field into its own column while owned_value is still alive. + *literal = VExprContext::create_shared(VLiteral::create_shared(column.type, value)); + return Status::OK(); +} + +bool has_shared_descendant_field_id(const ColumnDefinition& table, const ColumnDefinition& file) { + const auto& table_children = + table.identity_children.empty() ? table.children : table.identity_children; + for (const auto& table_child : table_children) { + if (!table_child.has_identifier_field_id()) { + continue; + } + const auto file_child = + std::ranges::find_if(file.children, [&](const ColumnDefinition& candidate) { + return candidate.has_identifier_field_id() && + candidate.get_identifier_field_id() == + table_child.get_identifier_field_id(); + }); + if (file_child != file.children.end() || + std::ranges::any_of(file.children, [&](const ColumnDefinition& candidate) { + return has_shared_descendant_field_id(table_child, candidate); + })) { + return true; + } + } + return false; +} + std::string mapping_mode_to_string(TableColumnMappingMode mode) { switch (mode) { case TableColumnMappingMode::BY_FIELD_ID: @@ -81,12 +148,16 @@ bool column_has_name(const ColumnDefinition& column, const std::string& name) { } bool column_names_match(const ColumnDefinition& lhs, const ColumnDefinition& rhs) { - if (column_has_name(rhs, lhs.name)) { - return true; - } - if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { - return true; + if (!lhs.has_name_mapping) { + if (column_has_name(rhs, lhs.name)) { + return true; + } + if (lhs.has_identifier_name() && column_has_name(rhs, lhs.get_identifier_name())) { + return true; + } } + // Explicit Iceberg name mapping is authoritative: an empty alias list represents a field that + // did not exist in the imported file, so only transported aliases may match. return std::ranges::any_of(lhs.name_mapping, [&](const std::string& alias) { return column_has_name(rhs, alias); }); @@ -322,18 +393,51 @@ static bool is_binary_comparison_predicate(const VExprSPtr& expr) { std::string TableColumnMapperOptions::debug_string() const { std::ostringstream out; - out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) << "}"; + out << "TableColumnMapperOptions{mode=" << mapping_mode_to_string(mode) + << ", allow_idless_complex_wrapper_projection=" << allow_idless_complex_wrapper_projection + << ", enable_row_lineage_virtual_columns=" << enable_row_lineage_virtual_columns << "}"; return out.str(); } +bool requires_char_or_varchar_truncation(const ColumnMapping& mapping) { + if (mapping.table_type == nullptr) { + return false; + } + const auto table_type = remove_nullable(mapping.table_type); + const auto primitive_type = table_type->get_primitive_type(); + if (primitive_type != TYPE_VARCHAR && primitive_type != TYPE_CHAR) { + return false; + } + const auto target_len = assert_cast(table_type.get())->len(); + if (target_len <= 0) { + return false; + } + if (mapping.file_type == nullptr) { + return true; + } + const auto file_type = remove_nullable(mapping.file_type); + DORIS_CHECK(file_type != nullptr); + int file_len = -1; + if (file_type->get_primitive_type() == TYPE_VARCHAR || + file_type->get_primitive_type() == TYPE_CHAR || + file_type->get_primitive_type() == TYPE_STRING) { + file_len = assert_cast(file_type.get())->len(); + } + return file_len < 0 || target_len < file_len; +} + std::string ColumnDefinition::debug_string() const { std::ostringstream out; out << "ColumnDefinition{name=" << name << ", identifier=" << field_debug_string(identifier) << ", name_mapping=" << join_debug_strings(name_mapping, [](const std::string& name) { return name; }) - << ", local_id=" << local_id << ", type=" << data_type_debug_string(type) << ", children=" + << ", has_name_mapping=" << has_name_mapping << ", local_id=" << local_id + << ", type=" << data_type_debug_string(type) << ", children=" << join_debug_strings(children, [](const ColumnDefinition& child) { return child.debug_string(); }) + << ", identity_children=" + << join_debug_strings(identity_children, + [](const ColumnDefinition& child) { return child.debug_string(); }) << ", has_default_expr=" << (default_expr != nullptr) << ", is_partition_key=" << is_partition_key << "}"; return out.str(); @@ -789,6 +893,19 @@ static bool collect_struct_element_chain(const VExprSPtr& expr, std::vector, table + // STRUCT, rows [NULL, 20], and `s.a > 10`, filtering in the file domain would drop + // NULL first and hide the table-contract violation. The reverse direction is safe: a required + // file value can always be wrapped as a nullable table value after filtering. + return !file_type->is_nullable() || table_type->is_nullable(); +} + static bool rewrite_struct_element_path_to_file_expr( const VExprSPtr& expr, const std::vector& mappings, const std::map& global_to_file_slot, @@ -815,6 +932,22 @@ static bool rewrite_struct_element_path_to_file_expr( return false; } + // Check every value-producing level, including the root struct. A nullable parent also makes + // a child access nullable even when the child type itself is required, so checking only the + // final leaf is insufficient. If any file level is more nullable than its table counterpart, + // keep the complete predicate above TableReader so schema validation observes all NULLs before + // row filtering. + if (!can_filter_before_table_nullability_alignment(rewrite_it->second.file_type, + rewrite_it->second.table_type)) { + return false; + } + for (size_t idx = 0; idx < struct_element_chain.size(); ++idx) { + if (!can_filter_before_table_nullability_alignment( + resolved.file_child_types[idx], struct_element_chain[idx]->data_type())) { + return false; + } + } + // File-local conjuncts are prepared against the file-reader Block, so both the root slot and // every struct selector must be expressed in file schema terms. For a renamed Iceberg field, // keeping the table selector would prepare `element_at(file_struct, 'renamed')` and @@ -843,6 +976,209 @@ static bool rewrite_struct_element_path_to_file_expr( return true; } +static VExprSPtr cast_file_expr_to_table_type(const VExprSPtr& file_expr, + const DataTypePtr& table_type, + RewriteContext* rewrite_context) { + DORIS_CHECK(file_expr != nullptr); + DORIS_CHECK(table_type != nullptr); + DORIS_CHECK(rewrite_context != nullptr); + auto cast_expr = Cast::create_shared(table_type); + cast_expr->add_child(file_expr); + rewrite_context->add_created_expr(cast_expr); + return cast_expr; +} + +// Prefer comparing in the physical file leaf type when a table predicate uses a promoted struct +// child. For example, with table STRUCT, old-file STRUCT, and `s.a = 10`, the +// localized predicate should be `file_s.a::INT = 10::INT`, not +// `CAST(file_s.a::INT AS BIGINT) = 10::BIGINT`. Converting one literal avoids a cast for every row. +// +// This rewrite is valid only when every possible file value survives file-to-table conversion and +// the particular literal survives a table-to-file-to-table round trip. A value such as BIGINT +// 2147483648 cannot be represented by an INT file leaf, so that case deliberately falls back to +// `CAST(file_s.a AS BIGINT) = 2147483648`, which preserves the original table-level semantics. +static bool rewrite_binary_struct_literal_predicate( + const VExprSPtr& expr, const std::vector& filter_mappings, + const std::map& global_to_file_slot, + RewriteContext* rewrite_context, bool* can_localize) { + DORIS_CHECK(can_localize != nullptr); + if (!is_binary_comparison_predicate(expr)) { + return false; + } + auto children = expr->children(); + int struct_child_idx = -1; + int literal_child_idx = -1; + if (is_struct_element_expr(children[0])) { + struct_child_idx = 0; + literal_child_idx = 1; + } else if (is_struct_element_expr(children[1])) { + struct_child_idx = 1; + literal_child_idx = 0; + } else { + return false; + } + + const auto table_leaf_type = children[struct_child_idx]->data_type(); + DORIS_CHECK(table_leaf_type != nullptr); + auto table_literal = unwrap_literal_for_file_cast(children[literal_child_idx], table_leaf_type); + if (table_literal == nullptr || + !rewrite_struct_element_path_to_file_expr(children[struct_child_idx], filter_mappings, + global_to_file_slot, rewrite_context)) { + return false; + } + + const auto file_leaf_type = children[struct_child_idx]->data_type(); + DORIS_CHECK(file_leaf_type != nullptr); + const FileSlotRewriteInfo leaf_rewrite_info { + .block_position = 0, + .file_type = file_leaf_type, + .table_type = table_leaf_type, + .file_column_name = {}, + }; + auto file_literal = + rewrite_literal_to_file_type(table_literal, leaf_rewrite_info, rewrite_context); + if (file_literal != nullptr) { + children[literal_child_idx] = std::move(file_literal); + } else { + if (!is_lossless_file_to_table_numeric_cast(file_leaf_type, table_leaf_type)) { + // A narrowing or otherwise lossy cast can fail or produce NULL while TableReader + // materializes the table schema. Evaluating it here could filter the offending row + // before that validation, so keep the complete predicate above TableReader. + *can_localize = false; + return true; + } + children[struct_child_idx] = cast_file_expr_to_table_type(children[struct_child_idx], + table_leaf_type, rewrite_context); + children[literal_child_idx] = original_table_literal(table_literal, rewrite_context); + } + expr->set_children(std::move(children)); + return true; +} + +// IN must use one comparison type for its probe and every candidate. Rewrite the complete literal +// set only when all values are exactly representable in the file leaf type; one unsafe value makes +// the whole predicate fall back to a table-type cast. For example, an INT file leaf can evaluate +// `BIGINT IN (10, 20)` as `INT IN (10, 20)`, but `BIGINT IN (10, 2147483648)` must stay BIGINT. +static bool rewrite_in_struct_literal_predicate( + const VExprSPtr& expr, const std::vector& filter_mappings, + const std::map& global_to_file_slot, + RewriteContext* rewrite_context, bool* can_localize) { + DORIS_CHECK(can_localize != nullptr); + if (expr->node_type() != TExprNodeType::IN_PRED || expr->get_num_children() < 2 || + !is_struct_element_expr(expr->children()[0])) { + return false; + } + auto children = expr->children(); + const auto table_leaf_type = children[0]->data_type(); + DORIS_CHECK(table_leaf_type != nullptr); + VExprSPtrs table_literals; + table_literals.reserve(children.size() - 1); + for (size_t child_idx = 1; child_idx < children.size(); ++child_idx) { + auto table_literal = unwrap_literal_for_file_cast(children[child_idx], table_leaf_type); + if (table_literal == nullptr) { + return false; + } + table_literals.push_back(std::move(table_literal)); + } + if (!rewrite_struct_element_path_to_file_expr(children[0], filter_mappings, global_to_file_slot, + rewrite_context)) { + return false; + } + + const auto file_leaf_type = children[0]->data_type(); + DORIS_CHECK(file_leaf_type != nullptr); + const FileSlotRewriteInfo leaf_rewrite_info { + .block_position = 0, + .file_type = file_leaf_type, + .table_type = table_leaf_type, + .file_column_name = {}, + }; + VExprSPtrs file_literals; + file_literals.reserve(table_literals.size()); + for (const auto& table_literal : table_literals) { + auto file_literal = + rewrite_literal_to_file_type(table_literal, leaf_rewrite_info, rewrite_context); + if (file_literal == nullptr) { + if (!is_lossless_file_to_table_numeric_cast(file_leaf_type, table_leaf_type)) { + *can_localize = false; + return true; + } + children[0] = + cast_file_expr_to_table_type(children[0], table_leaf_type, rewrite_context); + for (size_t literal_idx = 0; literal_idx < table_literals.size(); ++literal_idx) { + children[literal_idx + 1] = + original_table_literal(table_literals[literal_idx], rewrite_context); + } + expr->set_children(std::move(children)); + return true; + } + file_literals.push_back(std::move(file_literal)); + } + + for (size_t literal_idx = 0; literal_idx < file_literals.size(); ++literal_idx) { + children[literal_idx + 1] = std::move(file_literals[literal_idx]); + } + expr->set_children(std::move(children)); + return true; +} + +static VExprSPtr rewrite_struct_or_slot_expr_to_file_expr( + const VExprSPtr& expr, + const std::map& global_to_file_slot, + const std::vector& filter_mappings, RewriteContext* rewrite_context, + bool* can_localize) { + if (is_struct_element_expr(expr)) { + const auto table_leaf_type = expr->data_type(); + if (!rewrite_struct_element_path_to_file_expr(expr, filter_mappings, global_to_file_slot, + rewrite_context)) { + // The scanner still evaluates the original table-level conjunct after TableReader + // finalizes the output block. Skipping an unlocalizable file conjunct is therefore + // safer than preparing a partially rewritten expression against the wrong struct + // layout. In particular, do not generate file-local conjuncts for computed complex + // parents such as `element_at(element_at(map_values(m), 1), 'field')`; only direct + // slot-rooted struct chains are supported here. + *can_localize = false; + return expr; + } + DORIS_CHECK(table_leaf_type != nullptr); + DORIS_CHECK(expr->data_type() != nullptr); + if (!expr->data_type()->equals(*table_leaf_type)) { + if (!is_lossless_file_to_table_numeric_cast(expr->data_type(), table_leaf_type)) { + *can_localize = false; + return expr; + } + // Path localization changes the leaf to the physical file type. For example, after an + // Iceberg evolution from STRUCT to STRUCT, the localized old-file + // predicate is initially `element_at(file_col, 'a')::INT = 10::BIGINT`. Cast only the + // leaf back to BIGINT so the comparison has matching operands without forcing a cast + // of the entire evolved struct (whose children may also have been added or reordered). + return cast_file_expr_to_table_type(expr, table_leaf_type, rewrite_context); + } + return expr; + } + + DORIS_CHECK(expr->is_slot_ref()); + const auto* slot_ref = assert_cast(expr.get()); + const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*slot_ref)); + if (rewrite_it == global_to_file_slot.end()) { + return expr; + } + const auto& rewrite_info = rewrite_it->second; + auto file_slot = create_file_slot_ref(*slot_ref, rewrite_info, rewrite_context); + if (rewrite_info.file_type->equals(*rewrite_info.table_type)) { + return file_slot; + } + if (needs_complex_file_slot_cast(rewrite_info.file_type, rewrite_info.table_type)) { + // Generic file-local expressions cannot safely cast an evolved complex file slot back to + // the table type. For example, ARRAY_CONTAINS(MAP_KEYS(m), 'person5') only reads map keys, + // but CAST(file_m AS table_m) first forces an incompatible old value struct into the new + // layout. Keep such predicates at table level, after TableReader materializes evolution. + *can_localize = false; + return expr; + } + return cast_file_expr_to_table_type(file_slot, rewrite_info.table_type, rewrite_context); +} + static VExprSPtr rewrite_table_expr_to_file_expr( const VExprSPtr& expr, const std::map& global_to_file_slot, @@ -874,52 +1210,18 @@ static VExprSPtr rewrite_table_expr_to_file_expr( if (rewrite_in_slot_literal_predicate(expr, global_to_file_slot, rewrite_context)) { return expr; } - if (is_struct_element_expr(expr)) { - if (!rewrite_struct_element_path_to_file_expr(expr, filter_mappings, global_to_file_slot, - rewrite_context)) { - // The scanner still evaluates the original table-level conjunct after TableReader - // finalizes the output block. Skipping an unlocalizable file conjunct is therefore - // safer than preparing a partially rewritten expression against the wrong struct - // layout. In particular, do not generate file-local conjuncts for computed complex - // parents such as `element_at(element_at(map_values(m), 1), 'field')`; only direct - // slot-rooted struct chains are supported here. - *can_localize = false; - } + if (rewrite_binary_struct_literal_predicate(expr, filter_mappings, global_to_file_slot, + rewrite_context, can_localize)) { return expr; } - if (expr->is_slot_ref()) { - const auto* slot_ref = assert_cast(expr.get()); - const auto rewrite_it = global_to_file_slot.find(slot_ref_global_index(*slot_ref)); - if (rewrite_it != global_to_file_slot.end()) { - const auto& rewrite_info = rewrite_it->second; - auto file_slot = create_file_slot_ref(*slot_ref, rewrite_info, rewrite_context); - if (rewrite_info.file_type->equals(*rewrite_info.table_type)) { - return file_slot; - } - if (needs_complex_file_slot_cast(rewrite_info.file_type, rewrite_info.table_type)) { - // Generic file-local expressions cannot safely cast an evolved complex file slot - // back to the table type. Example: - // - // table filter: ARRAY_CONTAINS(MAP_KEYS(m), 'person5') - // old file: m MAP> - // table: m MAP> - // - // Although MAP_KEYS only reads the key column, wrapping the file slot as - // `CAST(file_m AS table_m)` forces the value struct cast first and fails because - // the old and new value structs have different fields. Keep such filters at the - // table level, where TableReader materializes the evolved complex value before - // Scanner evaluates the original conjunct. Direct slot-rooted struct child paths - // are handled by rewrite_struct_element_path_to_file_expr() above. - *can_localize = false; - return expr; - } - auto cast_expr = Cast::create_shared(rewrite_info.table_type); - cast_expr->add_child(std::move(file_slot)); - rewrite_context->add_created_expr(cast_expr); - return cast_expr; - } + if (rewrite_in_struct_literal_predicate(expr, filter_mappings, global_to_file_slot, + rewrite_context, can_localize)) { return expr; } + if (is_struct_element_expr(expr) || expr->is_slot_ref()) { + return rewrite_struct_or_slot_expr_to_file_expr(expr, global_to_file_slot, filter_mappings, + rewrite_context, can_localize); + } // The input is a split-local cloned tree. A previous split-local clone may already have // inserted Cast(slot). Keep that rewrite idempotent: rewrite the cast child from table slot to // the current split's file slot, and drop the cast when the current split no longer needs it. @@ -1729,7 +2031,12 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab mapping->global_index = global_index; mapping->table_column_name = table_column.name; mapping->table_type = table_column.type; - const auto row_lineage_type = row_lineage_virtual_column_type(table_column, _options.mode); + // Row-lineage names are Iceberg metadata contracts, not reserved names in generic Hive, + // Hudi, or Paimon schemas. Only the Iceberg reader may opt into virtual synthesis. + const auto row_lineage_type = + _options.enable_row_lineage_virtual_columns + ? row_lineage_virtual_column_type(table_column, _options.mode) + : TableVirtualColumnType::INVALID; if (const auto* partition_value = find_partition_value(table_column, _partition_values); table_column.is_partition_key && partition_value != nullptr) { // Partition values are split constants and must take precedence over defaults. @@ -1761,6 +2068,12 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab // Doris internal Iceberg row locator is never a physical Iceberg data column. It is built // from file path, row position and partition metadata for delete/update/merge. mapping->virtual_column_type = TableVirtualColumnType::ICEBERG_ROWID; + } else if (table_column.initial_default_value.has_value()) { + VExprContextSPtr initial_default; + RETURN_IF_ERROR(build_initial_default_literal(table_column, &initial_default)); + // Iceberg metadata is the authoritative logical value for files written before the field + // existed; the generic FE expression may still contain its Base64 transport text. + _set_constant_mapping(mapping, std::move(initial_default)); } else if (table_column.default_expr != nullptr) { // Missing schema-evolution column with an explicit default expression. _set_constant_mapping(mapping, table_column.default_expr); @@ -1875,11 +2188,12 @@ Status TableColumnMapper::_build_filter_entries(const FileScanRequest& file_requ Status TableColumnMapper::create_scan_request( const std::vector& table_filters, const std::vector& projected_columns, FileScanRequest* file_request, - RuntimeState* runtime_state) { + RuntimeState* runtime_state, FilterLocalizationResult* localization_result) { // FileReader evaluates expressions against a file-local block. This mapper owns the // table-column to file-column conversion, so it also owns the file-local block positions. file_request->predicate_columns.clear(); file_request->non_predicate_columns.clear(); + file_request->predicate_only_columns.clear(); file_request->local_positions.clear(); file_request->conjuncts.clear(); file_request->delete_conjuncts.clear(); @@ -1910,7 +2224,32 @@ Status TableColumnMapper::create_scan_request( // 2. Build referenced predicate columns // Hidden filter mappings must be built before localizing filters, so that they can be localized together with visible mappings and referenced by localized filter expressions. RETURN_IF_ERROR(_build_hidden_filter_mappings(table_filters)); - RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state)); + RETURN_IF_ERROR( + localize_filters(table_filters, file_request, runtime_state, localization_result)); + for (const auto& mapping : _hidden_mappings) { + if (!mapping.file_local_id.has_value()) { + continue; + } + const auto local_id = LocalColumnId(*mapping.file_local_id); + const bool is_visible_output = + std::ranges::any_of(_mappings, [local_id](const ColumnMapping& visible_mapping) { + return visible_mapping.file_local_id.has_value() && + LocalColumnId(*visible_mapping.file_local_id) == local_id; + }); + if (is_visible_output) { + continue; + } + // A localized predicate is enforced exactly before TableReader materializes output. Only + // truly hidden mappings are absent from the final table block and may discard their + // payload after that file-local evaluation. + if (std::ranges::any_of(file_request->predicate_columns, + [local_id](const LocalColumnIndex& projection) { + return projection.column_id() == local_id; + }) && + !file_request->is_predicate_only(local_id)) { + file_request->predicate_only_columns.push_back(local_id); + } + } // 3. Rebuild output projection expressions for projected columns. localize_filters() has // already applied the final scan projection to mapping.file_type/projected_file_children before // rewriting filter expressions. @@ -1951,7 +2290,12 @@ ColumnMapping* TableColumnMapper::_find_filter_mapping(GlobalIndex global_index) Status TableColumnMapper::localize_filters(const std::vector& table_filters, FileScanRequest* file_request, - RuntimeState* runtime_state) { + RuntimeState* runtime_state, + FilterLocalizationResult* localization_result) { + if (localization_result != nullptr) { + localization_result->localized_filters.assign(table_filters.size(), false); + } + std::set localized_predicate_columns; FilterProjectionMap filter_projections; auto filter_mappings = _filter_visible_mappings(); RETURN_IF_ERROR(build_nested_struct_filter_projection_map(table_filters, filter_mappings, @@ -1988,9 +2332,28 @@ Status TableColumnMapper::localize_filters(const std::vector& table // This keeps expression localization independent from filter iteration order. filter_mappings = _filter_visible_mappings(); const auto global_to_file_slot = build_file_slot_rewrite_map(filter_mappings, _filter_entries); - for (const auto& table_filter : table_filters) { - if (table_filter.conjunct != nullptr && - table_filter_has_only_local_entries(table_filter, _filter_entries)) { + for (size_t filter_index = 0; filter_index < table_filters.size(); ++filter_index) { + const auto& table_filter = table_filters[filter_index]; + if (table_filter.conjunct != nullptr && table_filter.conjunct->root() != nullptr) { + const auto root = table_filter.conjunct->root(); + const auto impl = root->get_impl(); + const auto predicate = impl != nullptr ? impl : root; + if (!table_filter.can_localize || !predicate->is_deterministic() || + !table_filter_has_only_local_entries(table_filter, _filter_entries)) { + continue; + } + if (runtime_state != nullptr && + runtime_state->query_options().truncate_char_or_varchar_columns && + std::ranges::any_of(table_filter.global_indices, [&](GlobalIndex global_index) { + const auto* mapping = _find_filter_mapping(global_index); + return mapping != nullptr && requires_char_or_varchar_truncation(*mapping); + })) { + // The table predicate observes the bounded value after finalize; evaluating it on + // a wider file string would change equality and range semantics. + continue; + } + // FileReader becomes the exact owner only for a stable predicate whose complete + // expression can be rewritten against this split's physical schema. RewriteContext rewrite_context {.runtime_state = runtime_state}; VExprSPtr rewrite_root; Status clone_status; @@ -2001,8 +2364,7 @@ Status TableColumnMapper::localize_filters(const std::vector& table // `element_at(MAP_VALUES(m)[1], 'age') > 30`. The current file-local rewrite only // understands top-level slots and struct-element paths rooted at top-level slots; // cloning such expressions can hit the generic TExpr complex-type limitation. - // Leave them above TableReader, where Scanner evaluates the original table-level - // conjunct after final materialization. + // Leave them for TableReader after final table-schema materialization. #ifndef NDEBUG return Status::InternalError( "Failed to clone table filter for file-local rewrite: {}, expr={}", @@ -2038,7 +2400,40 @@ Status TableColumnMapper::localize_filters(const std::vector& table auto localized_conjunct = VExprContext::create_shared(std::move(localized_root)); RETURN_IF_ERROR(rewrite_context.prepare_created_exprs(localized_conjunct.get())); file_request->conjuncts.push_back(std::move(localized_conjunct)); + if (localization_result != nullptr) { + localization_result->localized_filters[filter_index] = true; + } + for (const auto global_index : table_filter.global_indices) { + const auto* mapping = _find_filter_mapping(global_index); + if (mapping != nullptr && mapping->file_local_id.has_value() && + filter_conversion_has_local_source(mapping->filter_conversion)) { + localized_predicate_columns.emplace(*mapping->file_local_id); + } + } + } + } + + // Candidate columns are added before expression rewriting because their file-block positions + // are needed to localize slot refs. If rewriting rejects every filter that references a visible + // column, move its already-merged output/filter projection to the lazy non-predicate set + // instead of forcing it through the eager predicate path. + for (auto& mapping : _mappings) { + if (!mapping.file_local_id.has_value()) { + continue; + } + const auto local_id = LocalColumnId(*mapping.file_local_id); + if (localized_predicate_columns.contains(local_id)) { + continue; } + const auto predicate_it = std::ranges::find_if( + file_request->predicate_columns, [local_id](const LocalColumnIndex& projection) { + return projection.column_id() == local_id; + }); + if (predicate_it == file_request->predicate_columns.end()) { + continue; + } + file_request->non_predicate_columns.push_back(std::move(*predicate_it)); + file_request->predicate_columns.erase(predicate_it); } return Status::OK(); } @@ -2052,7 +2447,25 @@ const ColumnDefinition* TableColumnMapper::_find_file_field( }); return field_it == file_schema.end() ? nullptr : &*field_it; } - return matcher_for_mode(_options.mode).find(table_column, file_schema); + const auto* matched = matcher_for_mode(_options.mode).find(table_column, file_schema); + if (matched != nullptr || _options.mode != TableColumnMappingMode::BY_FIELD_ID || + !_options.allow_idless_complex_wrapper_projection || table_column.children.empty()) { + return matched; + } + const ColumnDefinition* wrapper = nullptr; + for (const auto& candidate : file_schema) { + if (candidate.has_identifier_field_id() || candidate.children.empty() || + !has_shared_descendant_field_id(table_column, candidate)) { + continue; + } + if (wrapper != nullptr) { + return nullptr; + } + wrapper = &candidate; + } + // Iceberg Parquet's PruneColumns retains an ID-less complex wrapper when a nested field ID is + // selected. Descendant IDs, not aliases, identify that wrapper; ambiguity remains unmapped. + return wrapper; } Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_column, @@ -2090,6 +2503,13 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c const auto* file_child = find_file_child_for_mapping(table_child, file_field, _options.mode, table_child_idx, synthesized_table_children); + if (file_child == nullptr && !synthesized_table_children && + _options.mode == TableColumnMappingMode::BY_FIELD_ID && + _options.allow_idless_complex_wrapper_projection) { + // Parquet can retain an ID-less wrapper at any depth when a selected descendant + // has an ID; apply the same opt-in fallback used for root lookup recursively. + file_child = _find_file_field(table_child, file_field.children); + } if (synthesized_table_children && file_child != nullptr) { const auto file_child_id = file_child->file_local_id(); if (std::ranges::find(synthesized_used_file_child_ids, file_child_id) != @@ -2115,6 +2535,10 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c child_mapping.table_type = table_child.type; child_mapping.file_type = table_child.type; child_mapping.filter_conversion = FilterConversionType::FINALIZE_ONLY; + // A missing nested field still has its Iceberg initial-default value in every row + // written before the field was added; carry it into recursive materialization. + RETURN_IF_ERROR(build_initial_default_column( + table_child, &child_mapping.initial_default_column)); mapping->child_mappings.push_back(std::move(child_mapping)); continue; } diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 4f417a680aad60..0860b687e4abc6 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -40,6 +40,13 @@ namespace doris::format { struct ColumnDefinition; struct TableFilter; +// Reports which table filters were fully rewritten into exact file-local predicates for the +// current split. The result is aligned with the TableFilter input vector and must not be reused for +// another split because schema evolution can change localization independently for every file. +struct FilterLocalizationResult { + std::vector localized_filters; +}; + enum class TableColumnMappingMode { // Match by ColumnDefinition::identifier TYPE_INT as field id. BY_FIELD_ID, @@ -150,16 +157,23 @@ struct ColumnMapping { FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; TableVirtualColumnType virtual_column_type = TableVirtualColumnType::INVALID; VExprContextSPtr default_expr; + // One-row constant owns variable-width payloads; Field is only a borrowed + // StringView and cannot safely outlive the Base64 decode buffer used to construct it. + ColumnPtr initial_default_column; std::string debug_string() const; }; struct TableColumnMapperOptions { TableColumnMappingMode mode = TableColumnMappingMode::BY_FIELD_ID; + bool allow_idless_complex_wrapper_projection = false; + bool enable_row_lineage_virtual_columns = false; std::string debug_string() const; }; +bool requires_char_or_varchar_truncation(const ColumnMapping& mapping); + Status clone_table_expr_tree(const VExprSPtr& expr, VExprSPtr* cloned_expr); const Field* find_partition_value(const ColumnDefinition& table_column, const std::map& partition_values); @@ -191,7 +205,8 @@ class TableColumnMapper { virtual Status create_scan_request(const std::vector& table_filters, const std::vector& projected_columns, FileScanRequest* file_request, - RuntimeState* runtime_state = nullptr); + RuntimeState* runtime_state = nullptr, + FilterLocalizationResult* localization_result = nullptr); // Localize table-level filters to the file schema. // Trivial mappings can copy structured predicates directly. Type changes may be localized with @@ -199,7 +214,8 @@ class TableColumnMapper { // table-level finalize/filter fallback. virtual Status localize_filters(const std::vector& table_filters, FileScanRequest* file_request, - RuntimeState* runtime_state = nullptr); + RuntimeState* runtime_state = nullptr, + FilterLocalizationResult* localization_result = nullptr); void clear() { _mappings.clear(); _hidden_mappings.clear(); diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp b/be/src/format_v2/delimited_text/delimited_text_reader.cpp index f19d12c75714b9..63486d174effe0 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.cpp +++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp @@ -38,6 +38,7 @@ #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/string_util.h" @@ -180,7 +181,9 @@ void DelimitedTextReader::_init_profile() { return; } - ADD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, 1); + file_scan_profile::ensure_hierarchy(_profile); + _text_profile.total_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, + file_scan_profile::FILE_READER, 1); _text_profile.open_file_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OpenFileTime", DELIMITED_TEXT_PROFILE, 1); _text_profile.create_line_reader_time = @@ -199,8 +202,10 @@ void DelimitedTextReader::_init_profile() { _profile, "RawLinesRead", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_read_before_filter = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsReadBeforeFilter", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + // RuntimeProfile counter names are global within one profile, so the format prefix preserves + // independent ownership when Parquet and delimited readers are initialized in either order. _text_profile.rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _profile, "DelimitedRowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_filtered_by_delete_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsFilteredByDeleteConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_returned = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -215,6 +220,7 @@ void DelimitedTextReader::_init_profile() { Status DelimitedTextReader::init(RuntimeState* state) { _init_profile(); + SCOPED_TIMER(_text_profile.total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("{} v2 reader requires scan params", _reader_name); @@ -275,6 +281,7 @@ Status DelimitedTextReader::init(RuntimeState* state) { } Status DelimitedTextReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_text_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("{} v2 file_schema is null", _reader_name); } @@ -288,6 +295,7 @@ std::unique_ptr DelimitedTextReader::create_column_mapper( } Status DelimitedTextReader::open(std::shared_ptr request) { + SCOPED_TIMER(_text_profile.total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); @@ -307,6 +315,7 @@ Status DelimitedTextReader::open(std::shared_ptr request) { } Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -358,11 +367,18 @@ Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& request, FileAggregateResult* result) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(result != nullptr); if (request.agg_type != TPushAggOp::type::COUNT) { return Status::NotSupported("{} v2 reader only supports COUNT aggregate pushdown", _reader_name); } + if (!request.columns.empty()) { + // Text files expose no NULL-count metadata, and this fast path intentionally skips field + // parsing. Returning the physical row count for COUNT(nullable_col) would be incorrect. + return Status::NotSupported("{} v2 reader cannot push down COUNT with a column argument", + _reader_name); + } if (_line_reader == nullptr) { return Status::InternalError("{} v2 reader is not open", _reader_name); } @@ -396,6 +412,7 @@ Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& req } Status DelimitedTextReader::close() { + SCOPED_TIMER(_text_profile.total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.h b/be/src/format_v2/delimited_text/delimited_text_reader.h index daea3bc90942d8..dff27980c9cd12 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.h +++ b/be/src/format_v2/delimited_text/delimited_text_reader.h @@ -59,6 +59,7 @@ class DelimitedTextReader : public FileReader { protected: struct DelimitedTextProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* open_file_time = nullptr; RuntimeProfile::Counter* create_line_reader_time = nullptr; RuntimeProfile::Counter* read_line_time = nullptr; diff --git a/be/src/format_v2/expr/equality_delete_predicate.cpp b/be/src/format_v2/expr/equality_delete_predicate.cpp index 4d5c511abdb181..1d111aa7a74930 100644 --- a/be/src/format_v2/expr/equality_delete_predicate.cpp +++ b/be/src/format_v2/expr/equality_delete_predicate.cpp @@ -26,28 +26,63 @@ #include "core/assert_cast.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "util/hash_util.hpp" namespace doris::format { namespace { bool column_value_equal(const ColumnPtr& lhs, size_t lhs_row, const ColumnPtr& rhs, size_t rhs_row) { - if (lhs->is_nullable() && rhs->is_nullable()) { - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; - } + const IColumn* lhs_data = lhs.get(); + const IColumn* rhs_data = rhs.get(); if (lhs->is_nullable()) { const auto& nullable_lhs = assert_cast(*lhs); - return !nullable_lhs.is_null_at(lhs_row) && - nullable_lhs.get_nested_column().compare_at(lhs_row, rhs_row, *rhs, -1) == 0; + if (nullable_lhs.is_null_at(lhs_row)) { + return rhs->is_nullable() && + assert_cast(*rhs).is_null_at(rhs_row); + } + lhs_data = &nullable_lhs.get_nested_column(); } if (rhs->is_nullable()) { const auto& nullable_rhs = assert_cast(*rhs); - return !nullable_rhs.is_null_at(rhs_row) && - lhs->compare_at(lhs_row, rhs_row, nullable_rhs.get_nested_column(), -1) == 0; + if (nullable_rhs.is_null_at(rhs_row)) { + return false; + } + rhs_data = &nullable_rhs.get_nested_column(); + } + const bool lhs_binary = check_and_get_column(*lhs_data) != nullptr || + check_and_get_column(*lhs_data) != nullptr; + const bool rhs_binary = check_and_get_column(*rhs_data) != nullptr || + check_and_get_column(*rhs_data) != nullptr; + if (lhs_binary && rhs_binary) { + // Iceberg schema evolution may represent the same byte key as STRING in one file and + // VARBINARY in another. Equality-delete semantics compare bytes, not column storage classes. + return lhs_data->get_data_at(lhs_row) == rhs_data->get_data_at(rhs_row); + } + return lhs_data->compare_at(lhs_row, rhs_row, *rhs_data, -1) == 0; +} + +void update_varbinary_hashes(const ColumnWithTypeAndName& entry, uint64_t* hashes) { + const IColumn* data = entry.column.get(); + const uint8_t* null_map = nullptr; + if (entry.column->is_nullable()) { + const auto& nullable = assert_cast(*entry.column); + data = &nullable.get_nested_column(); + null_map = nullable.get_null_map_data().data(); + } + for (size_t row = 0; row < entry.column->size(); ++row) { + if (null_map != nullptr && null_map[row] != 0) { + hashes[row] = HashUtil::xxHash64NullWithSeed(hashes[row]); + continue; + } + const auto bytes = data->get_data_at(row); + hashes[row] = HashUtil::xxHash64WithSeed(bytes.data, bytes.size, hashes[row]); } - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; } } // namespace @@ -155,8 +190,14 @@ ColumnPtr EqualityDeletePredicate::_evaluate_key_block(const Block& data_key_blo std::vector EqualityDeletePredicate::_build_hashes(const Block& block) { std::vector hashes(block.rows(), 0); - for (const auto& column : block.get_columns()) { - column->update_hashes_with_value(hashes.data(), nullptr); + for (const auto& entry : block) { + if (remove_nullable(entry.type)->get_primitive_type() == TYPE_VARBINARY) { + // ColumnVarbinary intentionally lacks the generic column hash hook. Keep the V2 delete + // hash byte-identical to ColumnString so schema-mapped binary keys share hash buckets. + update_varbinary_hashes(entry, hashes.data()); + } else { + entry.column->update_hashes_with_value(hashes.data(), nullptr); + } } return hashes; } diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp index 4f5b247c791efd..9bbf7c3a8fc066 100644 --- a/be/src/format_v2/file_reader.cpp +++ b/be/src/format_v2/file_reader.cpp @@ -53,6 +53,10 @@ std::string FileScanRequest::debug_string() const { << join_debug_strings( non_predicate_columns, [](const LocalColumnIndex& projection) { return projection.debug_string(); }) + << ", predicate_only_columns=" + << join_debug_strings( + predicate_only_columns, + [](LocalColumnId column_id) { return std::to_string(column_id.value()); }) << ", local_positions={"; size_t position_idx = 0; for (const auto& [column_id, block_position] : local_positions) { @@ -62,7 +66,14 @@ std::string FileScanRequest::debug_string() const { out << column_id << ":" << block_position; } out << "}, conjunct_count=" << conjuncts.size() - << ", delete_conjunct_count=" << delete_conjuncts.size() << "}"; + << ", delete_conjunct_count=" << delete_conjuncts.size() + << ", count_star_placeholder_columns={"; + const char* delimiter = ""; + for (const auto column_id : count_star_placeholder_columns) { + out << delimiter << column_id.value(); + delimiter = ","; + } + out << "}}"; return out.str(); } diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 59f684121ce1e3..1f90957064b254 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -70,13 +70,35 @@ struct FileScanRequest { // Columns read after row-level filtering. Predicate columns are also available for output and // should not be duplicated here. std::vector non_predicate_columns; + // Predicate columns introduced only to evaluate hidden filter slots. Their values are dead + // after all file-local predicates run, although the shared file block still needs row-shaped + // placeholders until TableReader finalizes projected columns. + std::vector predicate_only_columns; // file-local column id -> file-local output block position. std::map local_positions; - // Row-level filters converted to file-local expressions from table-level predicates. + // Row-level filters converted to file-local expressions from table-level predicates. Readers + // must enforce these exactly on returned rows; metadata pruning alone does not transfer + // predicate ownership away from TableReader. VExprContextSPtrs conjuncts; // Delete predicates converted to file-local expressions. A TRUE result means that the row is // deleted, so readers must invert each result when building their keep filter. VExprContextSPtrs delete_conjuncts; + // File-local ids retained only because Nereids keeps a minimum-width output tuple for an + // explicit COUNT(*). These columns have no semantic value: for example, after pruning a scan + // may retain an unsupported TIME_MILLIS leaf even though COUNT(*) only needs one row per + // surviving input row. A reader may synthesize defaults instead of reading a marked column + // while it remains non-predicate. If filters or equality deletes promote the same id to + // predicate_columns, the value is semantically required and must still be validated and read. + std::vector count_star_placeholder_columns; + + bool is_count_star_placeholder(LocalColumnId column_id) const { + return std::ranges::find(count_star_placeholder_columns, column_id) != + count_star_placeholder_columns.end(); + } + + bool is_predicate_only(LocalColumnId column_id) const { + return std::ranges::find(predicate_only_columns, column_id) != predicate_only_columns.end(); + } }; // Helper for constructing the scan-column layout in FileScanRequest. diff --git a/be/src/format_v2/jni/jdbc_reader.cpp b/be/src/format_v2/jni/jdbc_reader.cpp index e0391f3a13a8f0..41d3a5282b6255 100644 --- a/be/src/format_v2/jni/jdbc_reader.cpp +++ b/be/src/format_v2/jni/jdbc_reader.cpp @@ -34,17 +34,38 @@ namespace doris::format::jdbc { +Status validate_non_nullable_special_type_result(const IColumn& result, size_t rows) { + const auto* nullable = check_and_get_column(&result); + if (UNLIKELY(nullable == nullptr)) { + return Status::InternalError("JDBC special-type CAST did not return a nullable column"); + } + if (UNLIKELY(nullable->has_null(0, rows))) { + // CAST NULL represents invalid source data; stripping the null map would turn it into a + // valid-looking default for a NOT NULL destination. + return Status::DataQualityError( + "JDBC special-type CAST produced NULL for a non-nullable column"); + } + return Status::OK(); +} + std::string JdbcJniReader::connector_class() const { return "org/apache/doris/jdbc/JdbcJniScanner"; } Status JdbcJniReader::prepare_split(const format::SplitReadOptions& options) { - _jdbc_params.clear(); - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.table_format_type == "jdbc") { - _jdbc_params = std::map( - options.current_range.table_format_params.jdbc_params.begin(), - options.current_range.table_format_params.jdbc_params.end()); + { + // End these scopes before JniTableReader enters the same counters; nested use would count + // this JDBC parameter preparation twice instead of extending the common lifecycle total. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + _jdbc_params.clear(); + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.table_format_type == "jdbc") { + _jdbc_params = std::map( + options.current_range.table_format_params.jdbc_params.begin(), + options.current_range.table_format_params.jdbc_params.end()); + } } return format::JniTableReader::prepare_split(options); } @@ -177,6 +198,7 @@ Status JdbcJniReader::_cast_string_to_special_type(const format::JniTableReader: if (target_type->is_nullable()) { output_block->replace_by_position(column.output_index, result_column); } else { + RETURN_IF_ERROR(validate_non_nullable_special_type_result(*result_column, rows)); const auto* nullable_column = assert_cast(result_column.get()); output_block->replace_by_position(column.output_index, nullable_column->get_nested_column_ptr()); diff --git a/be/src/format_v2/jni/jdbc_reader.h b/be/src/format_v2/jni/jdbc_reader.h index 91a5878cb4622f..e10f8ad97902b8 100644 --- a/be/src/format_v2/jni/jdbc_reader.h +++ b/be/src/format_v2/jni/jdbc_reader.h @@ -29,6 +29,8 @@ namespace doris::format::jdbc { +Status validate_non_nullable_special_type_result(const IColumn& result, size_t rows); + class JdbcJniReader final : public format::JniTableReader { public: ~JdbcJniReader() override = default; diff --git a/be/src/format_v2/jni/jni_table_reader.cpp b/be/src/format_v2/jni/jni_table_reader.cpp index 3cb105193b56ae..7658a52fc86e0f 100644 --- a/be/src/format_v2/jni/jni_table_reader.cpp +++ b/be/src/format_v2/jni/jni_table_reader.cpp @@ -24,6 +24,7 @@ #include "core/block/block.h" #include "exprs/vexpr_context.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/string_util.h" @@ -31,17 +32,31 @@ namespace doris::format { Status JniTableReader::init(TableReadOptions&& options) { RETURN_IF_ERROR(TableReader::init(std::move(options))); - _init_profile(); + { + // Base and derived scopes must not overlap on the same counter: RuntimeProfile timers add + // deltas, so nested use would double-count instead of extending lifecycle coverage. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + _init_profile(); + } + SCOPED_TIMER(_connector_total_time); return Status::OK(); } Status JniTableReader::prepare_split(const SplitReadOptions& options) { - // EOF belongs to the previous split. Keep it set after closing that split so repeated reads - // are idempotent, and clear it only when a new split is explicitly prepared. - _eof = false; - _current_range = options.current_range; - RETURN_IF_ERROR(validate_scan_range(options.current_range)); + SCOPED_TIMER(_connector_total_time); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + // EOF belongs to the previous split. Keep it set after closing that split so repeated reads + // are idempotent, and clear it only when a new split is explicitly prepared. + _eof = false; + _current_range = options.current_range; + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + } RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -63,6 +78,9 @@ Status JniTableReader::prepare_split(const SplitReadOptions& options) { } Status JniTableReader::get_block(Block* output_block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.exec_timer); + SCOPED_TIMER(_connector_total_time); DORIS_CHECK(output_block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(output_block->columns() == _projected_columns.size()); @@ -109,7 +127,11 @@ Status JniTableReader::get_block(Block* output_block, bool* eos) { } Status JniTableReader::abort_split() { - RETURN_IF_ERROR(_close_jni_scanner()); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + RETURN_IF_ERROR(_close_jni_scanner()); + } return TableReader::abort_split(); } @@ -342,10 +364,16 @@ void JniTableReader::_publish_split_profile(JNIEnv* env) { } Status JniTableReader::close() { + SCOPED_TIMER(_connector_total_time); if (_closed) { return Status::OK(); } - auto close_status = _close_jni_scanner(); + Status close_status; + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + close_status = _close_jni_scanner(); + } auto table_status = TableReader::close(); if (close_status.ok() && !table_status.ok()) { close_status = std::move(table_status); @@ -535,7 +563,9 @@ void JniTableReader::_init_profile() { return; } const auto connector_name = _connector_name(); - ADD_TIMER(_scanner_profile, connector_name); + file_scan_profile::ensure_hierarchy(_scanner_profile); + _connector_total_time = + ADD_CHILD_TIMER(_scanner_profile, connector_name, file_scan_profile::TABLE_READER); _open_scanner_time = ADD_CHILD_TIMER(_scanner_profile, "OpenScannerTime", connector_name); _java_scan_time = ADD_CHILD_TIMER(_scanner_profile, "JavaScanTime", connector_name); _java_append_data_time = diff --git a/be/src/format_v2/jni/jni_table_reader.h b/be/src/format_v2/jni/jni_table_reader.h index 24f84ca8d62756..5e48270515f996 100644 --- a/be/src/format_v2/jni/jni_table_reader.h +++ b/be/src/format_v2/jni/jni_table_reader.h @@ -84,6 +84,7 @@ class JniTableReader : public TableReader { virtual Status _open_jni_scanner(); bool _reserve_split_profile_publication(); const std::vector& jni_columns() const { return _jni_columns; } + RuntimeProfile::Counter* connector_total_timer() const { return _connector_total_time; } TFileRangeDesc _current_range; private: @@ -110,6 +111,7 @@ class JniTableReader : public TableReader { bool _eof = false; bool _split_profile_published = false; + RuntimeProfile::Counter* _connector_total_time = nullptr; RuntimeProfile::Counter* _open_scanner_time = nullptr; RuntimeProfile::Counter* _java_scan_time = nullptr; RuntimeProfile::Counter* _java_append_data_time = nullptr; diff --git a/be/src/format_v2/jni/trino_connector_jni_reader.cpp b/be/src/format_v2/jni/trino_connector_jni_reader.cpp index 11c9945c5dea16..4f0a3f55c3891e 100644 --- a/be/src/format_v2/jni/trino_connector_jni_reader.cpp +++ b/be/src/format_v2/jni/trino_connector_jni_reader.cpp @@ -84,8 +84,15 @@ Status TrinoConnectorJniReader::validate_scan_range(const TFileRangeDesc& range) } Status TrinoConnectorJniReader::prepare_split(const format::SplitReadOptions& options) { - RETURN_IF_ERROR(validate_scan_range(options.current_range)); - RETURN_IF_ERROR(_set_spi_plugins_dir()); + { + // Plugin discovery can dominate a cold split. Use non-overlapping common scopes because + // the JNI base method subsequently enters the same RuntimeProfile counters. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + RETURN_IF_ERROR(_set_spi_plugins_dir()); + } return format::JniTableReader::prepare_split(options); } diff --git a/be/src/format_v2/json/json_reader.cpp b/be/src/format_v2/json/json_reader.cpp index 313115e82ce744..caf4205fa61c5b 100644 --- a/be/src/format_v2/json/json_reader.cpp +++ b/be/src/format_v2/json/json_reader.cpp @@ -47,6 +47,7 @@ #include "io/fs/stream_load_pipe.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/slice.h" @@ -174,7 +175,26 @@ JsonReader::~JsonReader() { static_cast(close()); } +void JsonReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* json_profile = "JsonReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, json_profile, file_scan_profile::FILE_READER, 1); + _open_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonOpenTime", json_profile, 1); + _read_document_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonReadDocumentTime", json_profile, 1); + _parse_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonParseTime", json_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonMaterializeTime", json_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonFilterTime", json_profile, 1); +} + Status JsonReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("JSON v2 reader requires scan params"); @@ -230,6 +250,7 @@ Status JsonReader::init(RuntimeState* state) { } Status JsonReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("JSON v2 file_schema is null"); } @@ -243,14 +264,22 @@ std::unique_ptr JsonReader::create_column_mapper( } Status JsonReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); _slot_name_to_index.clear(); + _hive_slot_name_to_index.clear(); _slot_name_to_index.reserve(_requested_columns.size()); + _hive_slot_name_to_index.reserve(_requested_columns.size()); for (size_t idx = 0; idx < _requested_columns.size(); ++idx) { auto name = _requested_columns[idx].slot_desc->col_name(); - _slot_name_to_index.emplace(_is_hive_table ? lower_key(name) : name, idx); + if (_is_hive_table) { + _hive_slot_name_to_index.emplace(std::move(name), idx); + } else { + _slot_name_to_index.emplace(std::move(name), idx); + } } _previous_positions.clear(); _reader_range = _json_range(); @@ -269,6 +298,7 @@ Status JsonReader::open(std::shared_ptr request) { } Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -297,7 +327,10 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { bool is_empty_row = false; Status st = Status::OK(); try { - st = _parse_next_json(&size, &_reader_eof); + { + SCOPED_TIMER(_parse_time); + st = _parse_next_json(&size, &_reader_eof); + } if (st.ok() && !_reader_eof) { if (size == 0) { is_empty_row = true; @@ -306,6 +339,7 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { } } if (st.ok() && !_reader_eof && !is_empty_row) { + SCOPED_TIMER(_materialize_time); st = _append_rows_from_current_value(file_block, &is_empty_row, &_reader_eof); } } catch (simdjson::simdjson_error& e) { @@ -324,13 +358,17 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } *eof = _reader_eof && *rows == 0; _eof = *eof; return Status::OK(); } Status JsonReader::close() { + SCOPED_TIMER(_total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); @@ -489,6 +527,7 @@ Status JsonReader::_parse_jsonpath_and_json_root() { } Status JsonReader::_read_one_document(size_t* size, bool* eof) { + SCOPED_TIMER(_read_document_time); DORIS_CHECK(size != nullptr); DORIS_CHECK(eof != nullptr); *size = 0; @@ -499,7 +538,9 @@ Status JsonReader::_read_one_document(size_t* size, bool* eof) { if (*eof) { return Status::OK(); } - _document_buffer.assign(reinterpret_cast(line), *size); + // The line reader owns this span until the next read, and parsing copies it immediately + // into the padded buffer. Borrowing it avoids a redundant line-sized string copy. + _document_view = std::string_view(reinterpret_cast(line), *size); return Status::OK(); } // Non-line mode treats the split as one JSON document. This supports a single object or an @@ -525,6 +566,7 @@ Status JsonReader::_read_one_document(size_t* size, bool* eof) { Slice result(_document_buffer.data(), _document_buffer.size()); RETURN_IF_ERROR(_physical_file_reader->read_at(_current_offset, result, size, _io_ctx.get())); _document_buffer.resize(*size); + _document_view = _document_buffer; if (*size == 0) { *eof = true; } @@ -539,6 +581,7 @@ Status JsonReader::_read_one_document_from_pipe(size_t* read_size) { DorisUniqueBufferPtr file_buf; RETURN_IF_ERROR(stream_load_pipe->read_one_message(&file_buf, read_size)); _document_buffer.assign(reinterpret_cast(file_buf.get()), *read_size); + _document_view = _document_buffer; if (!stream_load_pipe->is_chunked_transfer()) { return Status::OK(); } @@ -553,6 +596,7 @@ Status JsonReader::_read_one_document_from_pipe(size_t* read_size) { _document_buffer.append(reinterpret_cast(next_buf.get()), next_size); *read_size += next_size; } + _document_view = _document_buffer; return Status::OK(); } @@ -561,10 +605,10 @@ Status JsonReader::_parse_next_json(size_t* size, bool* eof) { if (*eof || *size == 0) { return Status::OK(); } - if (*size >= 3 && static_cast(_document_buffer[0]) == 0xEF && - static_cast(_document_buffer[1]) == 0xBB && - static_cast(_document_buffer[2]) == 0xBF) { - _document_buffer.erase(0, 3); + if (*size >= 3 && static_cast(_document_view[0]) == 0xEF && + static_cast(_document_view[1]) == 0xBB && + static_cast(_document_view[2]) == 0xBF) { + _document_view.remove_prefix(3); *size -= 3; } if (*size + simdjson::SIMDJSON_PADDING > _padded_size) { @@ -573,7 +617,7 @@ Status JsonReader::_parse_next_json(size_t* size, bool* eof) { } // Ondemand values reference the input buffer. Keep the padded bytes in a member buffer until the // current document is fully materialized. - std::memcpy(_padding_buffer.data(), _document_buffer.data(), *size); + std::memcpy(_padding_buffer.data(), _document_view.data(), *size); _original_doc_size = *size; const auto error = _json_parser->iterate(std::string_view(_padding_buffer.data(), *size), _padded_size) @@ -1087,32 +1131,39 @@ void JsonReader::_pop_back_last_inserted_value(Block* block, size_t column_index } size_t JsonReader::_column_index(std::string_view key, size_t key_index) { - std::string hive_key; - std::string_view lookup_key = key; - if (_is_hive_table) { - hive_key = lower_key(key); - lookup_key = hive_key; - } if (key_index < _previous_positions.size()) { // Most JSON lines share field order. Reuse the previous line's key-position mapping before // falling back to the hash table lookup. const auto previous = _previous_positions[key_index]; if (previous < _requested_columns.size()) { const auto previous_name = _requested_columns[previous].slot_desc->col_name(); - if ((_is_hive_table ? lower_key(previous_name) : previous_name) == lookup_key) { + if ((_is_hive_table && CaseInsensitiveStringEqual {}(previous_name, key)) || + (!_is_hive_table && previous_name == key)) { return previous; } } } - const auto it = _slot_name_to_index.find(std::string(lookup_key)); - if (it == _slot_name_to_index.end()) { + // Transparent lookup keeps the common per-key path allocation-free; Hive case folding is + // performed by the map's hash/equality without constructing a lower-cased string. + const auto index = _is_hive_table ? ([&]() -> std::optional { + const auto it = _hive_slot_name_to_index.find(key); + return it == _hive_slot_name_to_index.end() ? std::nullopt + : std::optional(it->second); + })() + : ([&]() -> std::optional { + const auto it = _slot_name_to_index.find(key); + return it == _slot_name_to_index.end() + ? std::nullopt + : std::optional(it->second); + })(); + if (!index.has_value()) { return static_cast(-1); } if (key_index >= _previous_positions.size()) { _previous_positions.resize(key_index + 1, static_cast(-1)); } - _previous_positions[key_index] = it->second; - return it->second; + _previous_positions[key_index] = *index; + return *index; } bool JsonReader::_is_root_path_for_column(const RequestedColumn& column) const { diff --git a/be/src/format_v2/json/json_reader.h b/be/src/format_v2/json/json_reader.h index 52cdfad6728d64..c7346cb1d66357 100644 --- a/be/src/format_v2/json/json_reader.h +++ b/be/src/format_v2/json/json_reader.h @@ -42,6 +42,40 @@ class IColumn; namespace doris::format::json { +struct TransparentStringHash { + using is_transparent = void; + size_t operator()(std::string_view value) const { + return std::hash {}(value); + } +}; + +struct CaseInsensitiveStringHash { + using is_transparent = void; + size_t operator()(std::string_view value) const { + size_t hash = 1469598103934665603ULL; + for (const unsigned char c : value) { + const unsigned char folded = c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; + hash = (hash ^ folded) * 1099511628211ULL; + } + return hash; + } +}; + +struct CaseInsensitiveStringEqual { + using is_transparent = void; + bool operator()(std::string_view lhs, std::string_view rhs) const { + if (lhs.size() != rhs.size()) return false; + for (size_t i = 0; i < lhs.size(); ++i) { + const unsigned char left = + lhs[i] >= 'A' && lhs[i] <= 'Z' ? lhs[i] + ('a' - 'A') : lhs[i]; + const unsigned char right = + rhs[i] >= 'A' && rhs[i] <= 'Z' ? rhs[i] + ('a' - 'A') : rhs[i]; + if (left != right) return false; + } + return true; + } +}; + // FileScannerV2 JSON reader. // // JSON files do not carry an embedded physical schema. The v2 table layer still needs a @@ -74,7 +108,12 @@ class JsonReader final : public FileReader { Status get_block(Block* file_block, size_t* rows, bool* eof) override; Status close() override; +#ifdef BE_TEST + size_t TEST_document_buffer_size() const { return _document_buffer.size(); } +#endif + private: + void _init_profile() override; // A requested column keeps both identities: // - `source_index`: index in FE file slots, used for jsonpaths and SerDe lookup. // - `block_position`: index in the caller's output block, used for materialization. @@ -134,9 +173,19 @@ class JsonReader final : public FileReader { TFileCompressType::type _range_compress_type = TFileCompressType::UNKNOWN; std::optional _stream_load_id; std::vector _requested_columns; - std::unordered_map _slot_name_to_index; + std::unordered_map> + _slot_name_to_index; + std::unordered_map + _hive_slot_name_to_index; std::vector _previous_positions; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_time = nullptr; + RuntimeProfile::Counter* _read_document_time = nullptr; + RuntimeProfile::Counter* _parse_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + io::FileReaderSPtr _physical_file_reader; std::unique_ptr _decompressor; std::unique_ptr _line_reader; @@ -170,6 +219,7 @@ class JsonReader final : public FileReader { simdjson::ondemand::array _array; simdjson::ondemand::array_iterator _array_iter; std::string _document_buffer; + std::string_view _document_view; std::string _padding_buffer; size_t _original_doc_size = 0; size_t _padded_size = 1024 * 1024 * 8 + simdjson::SIMDJSON_PADDING; diff --git a/be/src/format_v2/native/native_reader.cpp b/be/src/format_v2/native/native_reader.cpp index 0e348f3de2ccf5..3d5429e0a9a262 100644 --- a/be/src/format_v2/native/native_reader.cpp +++ b/be/src/format_v2/native/native_reader.cpp @@ -29,6 +29,7 @@ #include "format_v2/materialized_reader_util.h" #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/slice.h" @@ -54,7 +55,26 @@ NativeReader::~NativeReader() { static_cast(close()); } +void NativeReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* native_profile = "NativeReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, native_profile, file_scan_profile::FILE_READER, 1); + _read_block_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeReadBlockTime", native_profile, 1); + _deserialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeDeserializeTime", native_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeMaterializeTime", native_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeFilterTime", native_profile, 1); +} + Status NativeReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_file_description == nullptr) { return Status::InvalidArgument("Native v2 reader requires file description"); @@ -65,6 +85,7 @@ Status NativeReader::init(RuntimeState* state) { } Status NativeReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("Native v2 file_schema is null"); } @@ -79,6 +100,7 @@ std::unique_ptr NativeReader::create_column_mapper( } Status NativeReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); _first_block_consumed = false; @@ -88,6 +110,7 @@ Status NativeReader::open(std::shared_ptr request) { } Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -108,6 +131,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { if (_first_block_loaded && !_first_block_consumed) { buffer = _first_block_buffer; } else { + SCOPED_TIMER(_read_block_time); RETURN_IF_ERROR(_read_next_pblock(&buffer, &local_eof)); } @@ -131,11 +155,20 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { Block source_block; size_t uncompressed_bytes = 0; int64_t decompress_time = 0; - RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); - RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + { + SCOPED_TIMER(_deserialize_time); + RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); + } + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + } *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } if (_first_block_loaded && !_first_block_consumed) { _first_block_consumed = true; @@ -149,6 +182,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { } Status NativeReader::close() { + SCOPED_TIMER(_total_time); _file_reader.reset(); _tracing_file_reader.reset(); _request.reset(); @@ -231,6 +265,16 @@ Status NativeReader::_read_next_pblock(std::string* buffer, bool* eof) const { return Status::OK(); } + const auto remaining_prefix_bytes = _file_size - _current_offset; + if (remaining_prefix_bytes < sizeof(uint64_t)) { + // Header-only is the one valid empty-file representation and returned above. Once any + // prefix byte exists, all eight bytes are mandatory. For example, `header + 4 bytes` is a + // truncated Native file, not EOF that FileScannerV2 may skip as an empty split. + return Status::InternalError( + "truncated native block length in file {} at offset {}, expect {}, remaining {}", + _file_description->path, _current_offset, sizeof(uint64_t), remaining_prefix_bytes); + } + uint64_t block_len = 0; Slice len_slice(reinterpret_cast(&block_len), sizeof(block_len)); size_t bytes_read = 0; @@ -247,8 +291,22 @@ Status NativeReader::_read_next_pblock(std::string* buffer, bool* eof) const { } _current_offset += sizeof(block_len); if (block_len == 0) { - *eof = (_current_offset >= _file_size); - return Status::OK(); + // A header-only file reaches the `_current_offset >= _file_size` branch above and is a + // valid empty Native file. Once an explicit length prefix is present, however, zero is not + // an EOF marker in the Native format. For example, `header + uint64(0)` is truncated input, + // not a zero-row split, and must not escape as EOF for FileScannerV2 to count as empty. + return Status::InternalError("zero-length native block in file {} at offset {}", + _file_description->path, _current_offset - sizeof(block_len)); + } + + const auto remaining_block_bytes = cast_set(_file_size - _current_offset); + if (block_len > remaining_block_bytes) { + // Validate before allocating `block_len` bytes. Besides reporting a precise corruption + // error, this prevents a truncated file with a damaged length prefix from requesting an + // allocation much larger than the physical file. + return Status::InternalError( + "truncated native block body in file {} at offset {}, expect {}, remaining {}", + _file_description->path, _current_offset, block_len, remaining_block_bytes); } buffer->assign(block_len, '\0'); diff --git a/be/src/format_v2/native/native_reader.h b/be/src/format_v2/native/native_reader.h index 3719a6afd6c4f5..15a52fe6f8bc87 100644 --- a/be/src/format_v2/native/native_reader.h +++ b/be/src/format_v2/native/native_reader.h @@ -49,6 +49,7 @@ class NativeReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _validate_and_consume_header(); Status _ensure_schema_loaded() const; Status _read_next_pblock(std::string* buffer, bool* eof) const; @@ -65,6 +66,11 @@ class NativeReader final : public FileReader { mutable std::string _first_block_buffer; mutable bool _first_block_loaded = false; mutable bool _first_block_consumed = false; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _read_block_time = nullptr; + RuntimeProfile::Counter* _deserialize_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; }; } // namespace doris::format::native diff --git a/be/src/format_v2/orc/orc_file_input_stream.cpp b/be/src/format_v2/orc/orc_file_input_stream.cpp index a5144d7b2017b0..df0927aac3444a 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.cpp +++ b/be/src/format_v2/orc/orc_file_input_stream.cpp @@ -27,6 +27,7 @@ #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" #include "orc/Exceptions.hh" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "util/slice.h" @@ -54,24 +55,28 @@ class OrcMergedRangeFileReader final : public io::FileReader { _size(_file_reader->size()) { _statistics.apply_bytes += _range.end_offset - _range.start_offset; if (_profile != nullptr) { - const char* profile_name = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, profile_name, 1); - _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", profile_name, 1); - _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", profile_name, 1); - _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, + const char* profile_name = "OrcMergedSmallIO"; + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); + // RuntimeProfile counter lookup is flat, so every child must be format-qualified; + // a unique parent alone cannot prevent aliasing with Parquet's MergeRange reader. + _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedCopyTime", profile_name, 1); + _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedReadTime", profile_name, 1); + _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestIO", TUnit::UNIT, profile_name, 1); - _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedIO", TUnit::UNIT, + _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedIO", TUnit::UNIT, profile_name, 1); - _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, - profile_name, 1); - _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedBytes", TUnit::BYTES, + _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestBytes", + TUnit::BYTES, profile_name, 1); + _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedBytes", TUnit::BYTES, profile_name, 1); - _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ApplyBytes", TUnit::BYTES, - profile_name, 1); - _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OverReadBytes", TUnit::BYTES, - profile_name, 1); - _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ClusterNum", TUnit::UNIT, - profile_name, 1); + _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedApplyBytes", + TUnit::BYTES, profile_name, 1); + _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedOverReadBytes", + TUnit::BYTES, profile_name, 1); + _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedClusterNum", + TUnit::UNIT, profile_name, 1); } } @@ -113,6 +118,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { if (_profile == nullptr) { return; } + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -165,6 +171,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { OrcMergedRangeStatistics _statistics; RuntimeProfile::Counter* _copy_time = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; RuntimeProfile::Counter* _merged_io = nullptr; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index b728c6fbedb700..5248fcaf22dd11 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,7 @@ #include "common/config.h" #include "common/consts.h" #include "common/exception.h" +#include "common/logging.h" #include "core/block/block.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" @@ -74,10 +76,12 @@ #include "format_v2/timestamp_statistics.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/debug_points.h" #include "util/slice.h" #include "util/timezone_utils.h" @@ -139,6 +143,27 @@ bool is_orc_stop(const io::IOContext* io_ctx, const std::exception& e) { return io_ctx != nullptr && io_ctx->should_stop && std::string_view(e.what()) == "stop"; } +Status fall_back_without_orc_sarg(const char* operation, const std::string& file_path, + ::orc::RowReaderOptions* row_reader_options, + const std::exception& e) { + LOG(WARNING) << "Failed to " << operation << " ORC search argument for file " << file_path + << "; falling back to scan without SARG: " << e.what(); + row_reader_options->searchArgument(nullptr); + return Status::OK(); +} + +Status handle_orc_sarg_exception(const char* operation, const std::string& file_path, + ::orc::RowReaderOptions* row_reader_options, const Exception& e) { + if (e.code() == ErrorCode::MEM_ALLOC_FAILED) { + return Status::MemoryLimitExceeded("Failed to {} ORC search argument: {}", operation, + e.what()); + } + if (e.code() == ErrorCode::MEM_LIMIT_EXCEEDED) { + return e.to_status(); + } + return fall_back_without_orc_sarg(operation, file_path, row_reader_options, e); +} + bool is_hour_offset_timezone(std::string_view timezone) { return timezone.size() == 6 && (timezone[0] == '+' || timezone[0] == '-') && std::isdigit(static_cast(timezone[1])) && @@ -605,8 +630,12 @@ bool build_zone_map_from_orc_statistics(const ::orc::Type& type, return set_string_zone_map(type, statistics, zone_map); case ::orc::TypeKind::DATE: return set_date_zone_map(statistics, zone_map); - case ::orc::TypeKind::TIMESTAMP: - return set_timestamp_zone_map(statistics, timezone, false, zone_map); + case ::orc::TypeKind::TIMESTAMP: { + // ORC stores timestamp statistics as wall-clock values in UTC coordinates. Restore them + // with UTC so the session timezone does not shift the civil time. + static const auto utc_time_zone = cctz::utc_time_zone(); + return set_timestamp_zone_map(statistics, utc_time_zone, false, zone_map); + } case ::orc::TypeKind::TIMESTAMP_INSTANT: return set_timestamp_zone_map(statistics, timezone, enable_mapping_timestamp_tz, zone_map); case ::orc::TypeKind::DECIMAL: @@ -720,6 +749,7 @@ struct OrcReaderScanState { bool enable_filter_by_min_max = true; bool orc_lazy_read_enabled = false; bool orc_lazy_selection_valid = false; + OrcSargBuildOptions sarg_build_options; std::vector selected_stripe_ranges; size_t current_stripe_range = 0; @@ -747,7 +777,9 @@ void OrcReader::_init_profile() { } static const char* orc_profile = "OrcReader"; - ADD_TIMER_WITH_LEVEL(_profile, orc_profile, 1); + file_scan_profile::ensure_hierarchy(_profile); + _orc_profile.total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, orc_profile, file_scan_profile::FILE_READER, 1); _orc_profile.reader_call = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReaderCall", TUnit::UNIT, orc_profile, 1); _orc_profile.reader_inclusive_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -774,20 +806,22 @@ void OrcReader::_init_profile() { _profile, "EvaluatedRowGroupCount", TUnit::UNIT, orc_profile, 1); _orc_profile.read_row_count = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReadRowCount", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsFiltered", - TUnit::UNIT, orc_profile, 1); + // RuntimeProfile counter names are flat; format-qualified names keep ORC ownership stable when + // one scan profile also initializes Parquet counters in either order. + _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "OrcRowGroupsFiltered", TUnit::UNIT, orc_profile, 1); _orc_profile.filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); - _orc_profile.read_row_groups = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsReadNum", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredRowsByGroup", - TUnit::UNIT, orc_profile, 1); + _profile, "OrcRowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); + _orc_profile.read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcRowGroupsReadNum", + TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "OrcFilteredRowsByGroup", TUnit::UNIT, orc_profile, 1); _orc_profile.lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "FilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_bytes = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredBytes", TUnit::BYTES, orc_profile, 1); + _profile, "OrcFilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFilteredBytes", + TUnit::BYTES, orc_profile, 1); _orc_profile.open_file_num = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, orc_profile, 1); + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFileNum", TUnit::UNIT, orc_profile, 1); } void OrcReader::_collect_profile() const { @@ -845,6 +879,8 @@ format::ColumnDefinition OrcReader::row_position_column_definition() { } Status OrcReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_orc_profile.total_time); RETURN_IF_ERROR(format::FileReader::init(state)); _state = std::make_unique(); TimezoneUtils::find_cctz_time_zone(_state->timezone, _state->timezone_obj); @@ -853,6 +889,10 @@ Status OrcReader::init(RuntimeState* state) { _state->enable_filter_by_min_max = state->query_options().enable_orc_filter_by_min_max; _state->timezone = state->timezone(); _state->timezone_obj = state->timezone_obj(); + if (state->query_options().__isset.max_pushdown_conditions_per_column) { + _state->sarg_build_options.max_pushdown_conditions_per_column = + state->query_options().max_pushdown_conditions_per_column; + } } ::orc::ReaderOptions options; @@ -1129,6 +1169,7 @@ Status OrcReader::_fill_map_schema_children(const ::orc::Type& type, } Status OrcReader::get_schema(std::vector* const file_schema) const { + SCOPED_TIMER(_orc_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } @@ -1162,6 +1203,7 @@ std::unique_ptr OrcReader::create_column_mapper( } Status OrcReader::open(std::shared_ptr request) { + SCOPED_TIMER(_orc_profile.total_time); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); } @@ -1350,18 +1392,26 @@ Status OrcReader::_init_search_argument_from_local_filters() { if (conjunct == nullptr) { continue; } - has_pushdown = - build_orc_search_argument(*_request, *_state->root_type, _state->timezone_obj, - conjunct->root(), builder) || - has_pushdown; + has_pushdown = build_orc_search_argument( + *_request, *_state->root_type, _state->timezone_obj, + _state->sarg_build_options, conjunct->root(), builder) || + has_pushdown; } if (!has_pushdown) { return Status::OK(); } builder->end(); + DBUG_EXECUTE_IF("OrcReader._init_search_argument_from_local_filters.inject_failure", + DBUG_RUN_CALLBACK()); _state->row_reader_options.searchArgument(builder->build()); + } catch (const Exception& e) { + return handle_orc_sarg_exception("build", _file_description->path, + &_state->row_reader_options, e); + } catch (const std::bad_alloc& e) { + return Status::MemoryLimitExceeded("Failed to build ORC search argument: {}", e.what()); } catch (const std::exception& e) { - return Status::InternalError("Failed to build ORC search argument: {}", e.what()); + return fall_back_without_orc_sarg("build", _file_description->path, + &_state->row_reader_options, e); } return Status::OK(); } @@ -1421,6 +1471,8 @@ void OrcReader::_apply_split_range() { // ORC RowReader ranges are continuous, so non-adjacent surviving stripes are // compacted into separate ranges. +// Keep stripe selection and range state transitions atomic. +// NOLINTNEXTLINE(readability-function-size) Status OrcReader::_select_stripe_ranges_by_statistics() { _state->selected_stripe_ranges.clear(); _state->current_stripe_range = 0; @@ -1439,8 +1491,20 @@ Status OrcReader::_select_stripe_ranges_by_statistics() { std::vector sarg_needed_stripes; try { sarg_needed_stripes = _state->reader->getNeedReadStripes(_state->row_reader_options); + } catch (const Exception& e) { + if (is_orc_stop(_io_ctx.get(), e)) { + return Status::EndOfFile("stop"); + } + return handle_orc_sarg_exception("evaluate", _file_description->path, + &_state->row_reader_options, e); + } catch (const std::bad_alloc& e) { + return Status::MemoryLimitExceeded("Failed to evaluate ORC search argument: {}", e.what()); } catch (const std::exception& e) { - return Status::InternalError("Failed to evaluate ORC search argument: {}", e.what()); + if (is_orc_stop(_io_ctx.get(), e)) { + return Status::EndOfFile("stop"); + } + return fall_back_without_orc_sarg("evaluate", _file_description->path, + &_state->row_reader_options, e); } std::vector selected_stripes; @@ -1822,6 +1886,7 @@ Status OrcReader::_decode_column(const ::orc::Type& file_type, const ::orc::Type } Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -1935,14 +2000,23 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { } _state->orc_lazy_selection_valid = false; } else { - RETURN_IF_ERROR(_filter_block(file_block, rows)); + auto filter_status = _filter_block(file_block, rows); + if (!filter_status.ok()) { + // V2 evaluates residual predicates after ORC returns the batch, while callers retain + // the historical nextBatch error contract used to identify row-filter failures. + filter_status.prepend("Orc row reader nextBatch failed. reason = "); + return filter_status; + } } *eof = false; return Status::OK(); } +// COUNT and MIN/MAX share selected-stripe metadata and the same conservative fallback contract. +// NOLINTNEXTLINE(readability-function-size) Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(result != nullptr); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); @@ -2049,6 +2123,11 @@ Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& reque RETURN_IF_ERROR(find_projected_minmax_leaf(*_state->root_type, request_column.projection, &leaf_type)); DORIS_CHECK(leaf_type != nullptr); + if (leaf_type->getKind() == ::orc::TypeKind::TIMESTAMP && + _state->reader->getWriterVersion() < ::orc::WriterVersion_ORC_135) { + return Status::NotSupported( + "ORC TIMESTAMP min/max statistics are unsafe before writer version ORC-135"); + } auto& aggregate_column = result->columns[column_idx]; aggregate_column.projection = request_column.projection; @@ -2393,6 +2472,7 @@ void OrcReader::_filter_requested_columns(Block* file_block, const IColumn::Filt } Status OrcReader::close() { + SCOPED_TIMER(_orc_profile.total_time); _collect_profile(); if (_state != nullptr) { _state = std::make_unique(); diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index adba20fbce0b10..a67dc5ad7f5e3e 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -72,6 +72,7 @@ class OrcReader final : public format::FileReader { private: struct OrcProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* reader_call = nullptr; // ReaderCall RuntimeProfile::Counter* reader_inclusive_latency_us = nullptr; // ReaderInclusiveLatencyUs RuntimeProfile::Counter* decompression_call = nullptr; // DecompressionCall diff --git a/be/src/format_v2/orc/orc_search_argument.cpp b/be/src/format_v2/orc/orc_search_argument.cpp index 4766fa8304587a..244109c12ce6e6 100644 --- a/be/src/format_v2/orc/orc_search_argument.cpp +++ b/be/src/format_v2/orc/orc_search_argument.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -46,6 +47,9 @@ #include "exprs/vslot_ref.h" #include "exprs/vtopn_pred.h" #include "format_v2/timestamp_statistics.h" +#ifdef BE_TEST +#include "util/debug_points.h" +#endif namespace doris::format::orc { namespace { @@ -95,6 +99,24 @@ struct OrcSargComparison { TExprOpcode::type normalized_op = TExprOpcode::INVALID_OPCODE; }; +enum class OrcSargNodeKind { + AND, + OR, + NOT, + LESS_THAN, + LESS_THAN_EQUALS, + EQUALS, + IN, + IS_NULL, +}; + +struct OrcSargNode { + OrcSargNodeKind kind; + std::optional column; + std::vector<::orc::Literal> literals; + std::vector children; +}; + std::optional file_column_id_for_slot_position( const format::FileScanRequest& request, int slot_column_id) { if (slot_column_id < 0) { @@ -743,12 +765,8 @@ bool is_null_literal(const VExprSPtr& literal_expr) { literal->get_column_ptr()->is_null_at(0); } -// Buildability is checked before emission so the builder path can assume the -// expression shape was validated and use DORIS_CHECK for impossible branches. -bool can_build_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, - const cctz::time_zone& timezone, const VExprSPtr& expr); - -std::optional expression_for_search_argument(const VExprSPtr& expr) { +std::optional expression_for_search_argument(const VExprSPtr& expr, + const OrcSargBuildOptions& options) { if (expr == nullptr) { return std::nullopt; } @@ -766,6 +784,10 @@ std::optional expression_for_search_argument(const VExprSPtr& expr) { } if (const auto* direct_in = dynamic_cast(impl.get()); direct_in != nullptr) { + const auto filter_size = direct_in->get_set_func()->size(); + if (filter_size == 0 || filter_size > options.max_pushdown_conditions_per_column) { + return std::nullopt; + } VExprSPtr in_expr; if (!direct_in->get_slot_in_expr(in_expr)) { return std::nullopt; @@ -965,6 +987,9 @@ std::optional make_comparison_literal_for_sarg( std::optional> make_in_literals_for_sarg( const OrcSargColumn& column, const VExprSPtr& source_expr, const std::vector& children, const cctz::time_zone& timezone) { +#ifdef BE_TEST + DBUG_EXECUTE_IF("OrcSearchArgument.make_in_literals_for_sarg", DBUG_RUN_CALLBACK()); +#endif if (children.size() < 2) { return std::nullopt; } @@ -1080,35 +1105,6 @@ std::optional sarg_comparison_for_expr(const format::FileScan }; } -bool can_build_slot_literal_predicate(const format::FileScanRequest& request, - const ::orc::Type& root_type, const cctz::time_zone& timezone, - const VExprSPtr& expr) { - if (expr == nullptr || expr->children().size() != 2) { - return false; - } - return sarg_comparison_for_expr(request, root_type, timezone, expr).has_value(); -} - -bool can_build_in_predicate(const format::FileScanRequest& request, const ::orc::Type& root_type, - const cctz::time_zone& timezone, const VExprSPtr& expr) { - if (expr == nullptr || expr->children().size() < 2) { - return false; - } - const auto sarg_column = - sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[0]); - if (!sarg_column.has_value()) { - return false; - } - return make_in_literals_for_sarg(*sarg_column, expr->children()[0], expr->children(), timezone) - .has_value(); -} - -bool can_build_is_null_predicate(const format::FileScanRequest& request, - const ::orc::Type& root_type, const VExprSPtr& expr) { - return expr != nullptr && expr->children().size() == 1 && - sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[0]).has_value(); -} - std::optional sarg_column_for_null_safe_equal_null( const format::FileScanRequest& request, const ::orc::Type& root_type, const VExprSPtr& expr) { @@ -1170,263 +1166,356 @@ std::optional sarg_comparison_for_null_safe_equal_literal( return comparison_if_other_child_is_non_null_literal(1, 0); } -bool can_build_null_safe_equal_predicate(const format::FileScanRequest& request, - const ::orc::Type& root_type, - const cctz::time_zone& timezone, const VExprSPtr& expr) { - return sarg_column_for_null_safe_equal_null(request, root_type, expr).has_value() || - sarg_comparison_for_null_safe_equal_literal(request, root_type, timezone, expr) - .has_value(); -} - -bool contains_null_safe_equal(const VExprSPtr& expr) { - const auto sarg_expr = expression_for_search_argument(expr); - if (!sarg_expr.has_value() || *sarg_expr == nullptr) { - return false; - } - if (sarg_expr->get() != expr.get()) { - return contains_null_safe_equal(*sarg_expr); - } - if ((*sarg_expr)->op() == TExprOpcode::EQ_FOR_NULL) { - return true; - } - return std::ranges::any_of((*sarg_expr)->children(), contains_null_safe_equal); +OrcSargNode make_not_node(OrcSargNode child) { + std::vector children; + children.push_back(std::move(child)); + return OrcSargNode { + .kind = OrcSargNodeKind::NOT, + .column = std::nullopt, + .literals = {}, + .children = std::move(children), + }; } -bool can_build_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, - const cctz::time_zone& timezone, const VExprSPtr& expr) { - const auto sarg_expr = expression_for_search_argument(expr); - if (!sarg_expr.has_value()) { - return false; - } - if (*sarg_expr == nullptr) { - return false; - } - if (sarg_expr->get() != expr.get()) { - return can_build_search_argument(request, root_type, timezone, *sarg_expr); - } - - switch ((*sarg_expr)->op()) { - case TExprOpcode::COMPOUND_AND: - return std::ranges::any_of((*sarg_expr)->children(), [&](const auto& child) { - return can_build_search_argument(request, root_type, timezone, child); - }); - case TExprOpcode::COMPOUND_OR: - if (contains_null_safe_equal(*sarg_expr)) { - return false; - } - return !(*sarg_expr)->children().empty() && - std::ranges::all_of((*sarg_expr)->children(), [&](const auto& child) { - return can_build_search_argument(request, root_type, timezone, child); - }); - case TExprOpcode::COMPOUND_NOT: - if (contains_null_safe_equal(*sarg_expr)) { - return false; - } - return (*sarg_expr)->children().size() == 1 && - can_build_search_argument(request, root_type, timezone, (*sarg_expr)->children()[0]); +OrcSargNode make_comparison_node(OrcSargComparison comparison) { + OrcSargNodeKind kind; + bool negate = false; + switch (comparison.normalized_op) { case TExprOpcode::GE: + kind = OrcSargNodeKind::LESS_THAN; + negate = true; + break; case TExprOpcode::GT: + kind = OrcSargNodeKind::LESS_THAN_EQUALS; + negate = true; + break; case TExprOpcode::LE: + kind = OrcSargNodeKind::LESS_THAN_EQUALS; + break; case TExprOpcode::LT: + kind = OrcSargNodeKind::LESS_THAN; + break; case TExprOpcode::EQ: + kind = OrcSargNodeKind::EQUALS; + break; case TExprOpcode::NE: - return (*sarg_expr)->node_type() != TExprNodeType::NULL_AWARE_BINARY_PRED && - can_build_slot_literal_predicate(request, root_type, timezone, *sarg_expr); - case TExprOpcode::EQ_FOR_NULL: - return can_build_null_safe_equal_predicate(request, root_type, timezone, *sarg_expr); - case TExprOpcode::FILTER_IN: - case TExprOpcode::FILTER_NOT_IN: - return (*sarg_expr)->node_type() != TExprNodeType::NULL_AWARE_IN_PRED && - can_build_in_predicate(request, root_type, timezone, *sarg_expr); - case TExprOpcode::INVALID_OPCODE: - return (*sarg_expr)->node_type() == TExprNodeType::FUNCTION_CALL && - ((*sarg_expr)->fn().name.function_name == "is_null_pred" || - (*sarg_expr)->fn().name.function_name == "is_not_null_pred") && - can_build_is_null_predicate(request, root_type, *sarg_expr); + kind = OrcSargNodeKind::EQUALS; + negate = true; + break; default: - return false; + DORIS_CHECK(false) << "Unsupported normalized ORC SARG comparison op " + << comparison.normalized_op; + __builtin_unreachable(); } -} -void build_less_than(const OrcSargColumn& column, const ::orc::Literal& literal, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - builder->lessThan(column.column_id, column.predicate_type, literal); + std::vector<::orc::Literal> literals; + literals.push_back(std::move(comparison.literal)); + OrcSargNode node { + .kind = kind, + .column = comparison.column, + .literals = std::move(literals), + .children = {}, + }; + if (negate) { + return make_not_node(std::move(node)); + } + return node; } -void build_less_than(const OrcSargComparison& comparison, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - build_less_than(comparison.column, comparison.literal, builder); +OrcSargNode make_in_node(OrcSargColumn column, std::vector<::orc::Literal> literals) { + DORIS_CHECK(!literals.empty()); + return OrcSargNode { + .kind = literals.size() == 1 ? OrcSargNodeKind::EQUALS : OrcSargNodeKind::IN, + .column = column, + .literals = std::move(literals), + .children = {}, + }; } -void build_less_than_equals(const OrcSargColumn& column, const ::orc::Literal& literal, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - builder->lessThanEquals(column.column_id, column.predicate_type, literal); +OrcSargNode make_is_null_node(OrcSargColumn column) { + return OrcSargNode { + .kind = OrcSargNodeKind::IS_NULL, + .column = column, + .literals = {}, + .children = {}, + }; } -void build_less_than_equals(const OrcSargComparison& comparison, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - build_less_than_equals(comparison.column, comparison.literal, builder); -} +class OrcSargCompiler { +public: + OrcSargCompiler(const format::FileScanRequest& request, const ::orc::Type& root_type, + const cctz::time_zone& timezone, const OrcSargBuildOptions& options) + : _request(request), _root_type(root_type), _timezone(timezone), _options(options) {} -void build_equals(const OrcSargColumn& column, const ::orc::Literal& literal, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - builder->equals(column.column_id, column.predicate_type, literal); -} + std::optional compile(const VExprSPtr& expr) { return compile_node(expr, false); } -void build_equals(const OrcSargComparison& comparison, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - build_equals(comparison.column, comparison.literal, builder); -} +private: + std::optional normalized_expression(const VExprSPtr& expr) { + if (expr == nullptr) { + return std::nullopt; + } + const auto found = _normalized_expressions.find(expr.get()); + if (found != _normalized_expressions.end()) { + return found->second; + } + auto normalized = expression_for_search_argument(expr, _options); + _normalized_expressions.emplace(expr.get(), normalized); + return normalized; + } -void build_comparison_predicate(const format::FileScanRequest& request, - const ::orc::Type& root_type, const VExprSPtr& expr, - const cctz::time_zone& timezone, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - const auto comparison = *sarg_comparison_for_expr(request, root_type, timezone, expr); - switch (comparison.normalized_op) { - case TExprOpcode::GE: - builder->startNot(); - build_less_than(comparison, builder); - builder->end(); - return; - case TExprOpcode::GT: - builder->startNot(); - build_less_than_equals(comparison, builder); - builder->end(); - return; - case TExprOpcode::LE: - build_less_than_equals(comparison, builder); - return; - case TExprOpcode::LT: - build_less_than(comparison, builder); - return; - case TExprOpcode::EQ: - build_equals(comparison, builder); - return; - case TExprOpcode::NE: - builder->startNot(); - build_equals(comparison, builder); - builder->end(); - return; - default: - DORIS_CHECK(false) << "Unsupported normalized ORC SARG comparison op " - << comparison.normalized_op; + bool contains_null_safe_equal(const VExprSPtr& expr) { + const auto normalized = normalized_expression(expr); + if (!normalized.has_value() || *normalized == nullptr) { + return false; + } + if (normalized->get() != expr.get()) { + return contains_null_safe_equal(*normalized); + } + if ((*normalized)->op() == TExprOpcode::EQ_FOR_NULL) { + return true; + } + // The probe expression may be complex, but valid IN values are literals. Avoid walking a + // potentially huge literal list only for null-safe-equality metadata. + if ((*normalized)->op() == TExprOpcode::FILTER_IN || + (*normalized)->op() == TExprOpcode::FILTER_NOT_IN) { + return !(*normalized)->children().empty() && + contains_null_safe_equal((*normalized)->children().front()); + } + return std::ranges::any_of((*normalized)->children(), [this](const auto& child) { + return contains_null_safe_equal(child); + }); } -} -void build_in_predicate(const format::FileScanRequest& request, const ::orc::Type& root_type, - const VExprSPtr& expr, const cctz::time_zone& timezone, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - const auto sarg_column = - *sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[0]); - auto literals = *make_in_literals_for_sarg(sarg_column, expr->children()[0], expr->children(), - timezone); - DORIS_CHECK(!literals.empty()); - if (literals.size() == 1) { - builder->equals(sarg_column.column_id, sarg_column.predicate_type, literals.front()); - return; + std::optional compile_node(const VExprSPtr& expr, bool is_negated) { + const auto normalized = normalized_expression(expr); + if (!normalized.has_value() || *normalized == nullptr) { + return std::nullopt; + } + if (normalized->get() != expr.get()) { + return compile_node(*normalized, is_negated); + } + + switch ((*normalized)->op()) { + case TExprOpcode::COMPOUND_AND: + return compile_and(*normalized, is_negated); + case TExprOpcode::COMPOUND_OR: + return compile_or(*normalized, is_negated); + case TExprOpcode::COMPOUND_NOT: + return compile_not(*normalized, is_negated); + case TExprOpcode::GE: + case TExprOpcode::GT: + case TExprOpcode::LE: + case TExprOpcode::LT: + case TExprOpcode::EQ: + case TExprOpcode::NE: + return compile_comparison(*normalized); + case TExprOpcode::EQ_FOR_NULL: + return compile_null_safe_equal(*normalized); + case TExprOpcode::FILTER_IN: + case TExprOpcode::FILTER_NOT_IN: + return compile_in(*normalized); + case TExprOpcode::INVALID_OPCODE: + return compile_is_null(*normalized); + default: + return std::nullopt; + } } - builder->in(sarg_column.column_id, sarg_column.predicate_type, literals); -} -void build_is_null(const format::FileScanRequest& request, const ::orc::Type& root_type, - const VExprSPtr& expr, std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - const auto sarg_column = - *sarg_column_for_slot_or_safe_cast(request, root_type, expr->children()[0]); - builder->isNull(sarg_column.column_id, sarg_column.predicate_type); -} + std::optional compile_and(const VExprSPtr& expr, bool is_negated) { + const auto& source_children = expr->children(); + if (source_children.empty()) { + return std::nullopt; + } + std::vector children; + children.reserve(source_children.size()); + bool all_children_compiled = true; + for (const auto& child : source_children) { + auto compiled_child = compile_node(child, is_negated); + if (compiled_child.has_value()) { + children.push_back(std::move(*compiled_child)); + } else { + all_children_compiled = false; + } + } + // Omitting an AND child weakens the SARG only in positive context. Under an odd number of + // NOTs it would strengthen the SARG and could incorrectly prune matching stripes. + if (children.empty() || (is_negated && !all_children_compiled)) { + return std::nullopt; + } + return OrcSargNode { + .kind = OrcSargNodeKind::AND, + .column = std::nullopt, + .literals = {}, + .children = std::move(children), + }; + } -void build_null_safe_equal(const format::FileScanRequest& request, const ::orc::Type& root_type, - const VExprSPtr& expr, const cctz::time_zone& timezone, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - const auto sarg_column = sarg_column_for_null_safe_equal_null(request, root_type, expr); - if (sarg_column.has_value()) { - builder->isNull(sarg_column->column_id, sarg_column->predicate_type); - return; + std::optional compile_or(const VExprSPtr& expr, bool is_negated) { + const auto& source_children = expr->children(); + if (source_children.empty() || contains_null_safe_equal(expr)) { + return std::nullopt; + } + std::vector children; + children.reserve(source_children.size()); + for (const auto& child : source_children) { + auto compiled_child = compile_node(child, is_negated); + if (!compiled_child.has_value()) { + return std::nullopt; + } + children.push_back(std::move(*compiled_child)); + } + return OrcSargNode { + .kind = OrcSargNodeKind::OR, + .column = std::nullopt, + .literals = {}, + .children = std::move(children), + }; } - const auto comparison = - *sarg_comparison_for_null_safe_equal_literal(request, root_type, timezone, expr); - build_equals(comparison, builder); -} -bool build_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, - const cctz::time_zone& timezone, const VExprSPtr& expr, - std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - const auto sarg_expr = expression_for_search_argument(expr); - if (!sarg_expr.has_value() || *sarg_expr == nullptr) { - return false; + std::optional compile_not(const VExprSPtr& expr, bool is_negated) { + if (expr->children().size() != 1 || contains_null_safe_equal(expr)) { + return std::nullopt; + } + auto child = compile_node(expr->children()[0], !is_negated); + if (!child.has_value()) { + return std::nullopt; + } + return make_not_node(std::move(*child)); } - if (sarg_expr->get() != expr.get()) { - return build_search_argument(request, root_type, timezone, *sarg_expr, builder); + + std::optional compile_comparison(const VExprSPtr& expr) const { + if (expr->node_type() == TExprNodeType::NULL_AWARE_BINARY_PRED) { + return std::nullopt; + } + auto comparison = sarg_comparison_for_expr(_request, _root_type, _timezone, expr); + if (!comparison.has_value()) { + return std::nullopt; + } + return make_comparison_node(std::move(*comparison)); } - if (!can_build_search_argument(request, root_type, timezone, *sarg_expr)) { - return false; + + std::optional compile_null_safe_equal(const VExprSPtr& expr) const { + const auto column = sarg_column_for_null_safe_equal_null(_request, _root_type, expr); + if (column.has_value()) { + return make_is_null_node(*column); + } + auto comparison = + sarg_comparison_for_null_safe_equal_literal(_request, _root_type, _timezone, expr); + if (!comparison.has_value()) { + return std::nullopt; + } + return make_comparison_node(std::move(*comparison)); } - switch ((*sarg_expr)->op()) { - case TExprOpcode::COMPOUND_AND: - builder->startAnd(); - for (const auto& child : (*sarg_expr)->children()) { - static_cast(build_search_argument(request, root_type, timezone, child, builder)); + std::optional compile_in(const VExprSPtr& expr) const { + if (expr->node_type() == TExprNodeType::NULL_AWARE_IN_PRED || expr->children().size() < 2) { + return std::nullopt; } - builder->end(); - return true; - case TExprOpcode::COMPOUND_OR: - builder->startOr(); - for (const auto& child : (*sarg_expr)->children()) { - const auto built = build_search_argument(request, root_type, timezone, child, builder); - DORIS_CHECK(built); + const auto column = + sarg_column_for_slot_or_safe_cast(_request, _root_type, expr->children()[0]); + if (!column.has_value()) { + return std::nullopt; } - builder->end(); - return true; - case TExprOpcode::COMPOUND_NOT: - builder->startNot(); - DORIS_CHECK(build_search_argument(request, root_type, timezone, (*sarg_expr)->children()[0], - builder)); - builder->end(); - return true; - case TExprOpcode::GE: - case TExprOpcode::GT: - case TExprOpcode::LE: - case TExprOpcode::LT: - case TExprOpcode::EQ: - case TExprOpcode::NE: - build_comparison_predicate(request, root_type, *sarg_expr, timezone, builder); - return true; - case TExprOpcode::EQ_FOR_NULL: - build_null_safe_equal(request, root_type, *sarg_expr, timezone, builder); - return true; - case TExprOpcode::FILTER_IN: - build_in_predicate(request, root_type, *sarg_expr, timezone, builder); - return true; - case TExprOpcode::FILTER_NOT_IN: - builder->startNot(); - build_in_predicate(request, root_type, *sarg_expr, timezone, builder); - builder->end(); - return true; - case TExprOpcode::INVALID_OPCODE: - if ((*sarg_expr)->fn().name.function_name == "is_null_pred") { - build_is_null(request, root_type, *sarg_expr, builder); - return true; + auto literals = make_in_literals_for_sarg(*column, expr->children()[0], expr->children(), + _timezone); + if (!literals.has_value()) { + return std::nullopt; } - if ((*sarg_expr)->fn().name.function_name == "is_not_null_pred") { - builder->startNot(); - build_is_null(request, root_type, *sarg_expr, builder); - builder->end(); - return true; + auto node = make_in_node(*column, std::move(*literals)); + if (expr->op() == TExprOpcode::FILTER_NOT_IN) { + node = make_not_node(std::move(node)); } - return false; - default: - return false; + return node; + } + + std::optional compile_is_null(const VExprSPtr& expr) const { + if (expr->node_type() != TExprNodeType::FUNCTION_CALL || expr->children().size() != 1) { + return std::nullopt; + } + const auto& function_name = expr->fn().name.function_name; + if (function_name != "is_null_pred" && function_name != "is_not_null_pred") { + return std::nullopt; + } + const auto column = + sarg_column_for_slot_or_safe_cast(_request, _root_type, expr->children()[0]); + if (!column.has_value()) { + return std::nullopt; + } + auto node = make_is_null_node(*column); + if (function_name == "is_not_null_pred") { + node = make_not_node(std::move(node)); + } + return node; + } + + const format::FileScanRequest& _request; + const ::orc::Type& _root_type; + const cctz::time_zone& _timezone; + const OrcSargBuildOptions& _options; + std::unordered_map> _normalized_expressions; +}; + +void emit_search_argument(const OrcSargNode& node, ::orc::SearchArgumentBuilder& builder) { + switch (node.kind) { + case OrcSargNodeKind::AND: + DORIS_CHECK(!node.children.empty()); + builder.startAnd(); + for (const auto& child : node.children) { + emit_search_argument(child, builder); + } + builder.end(); + return; + case OrcSargNodeKind::OR: + DORIS_CHECK(!node.children.empty()); + builder.startOr(); + for (const auto& child : node.children) { + emit_search_argument(child, builder); + } + builder.end(); + return; + case OrcSargNodeKind::NOT: + DORIS_CHECK(node.children.size() == 1); + builder.startNot(); + emit_search_argument(node.children.front(), builder); + builder.end(); + return; + case OrcSargNodeKind::LESS_THAN: + DORIS_CHECK(node.column.has_value() && node.literals.size() == 1); + builder.lessThan(node.column->column_id, node.column->predicate_type, + node.literals.front()); + return; + case OrcSargNodeKind::LESS_THAN_EQUALS: + DORIS_CHECK(node.column.has_value() && node.literals.size() == 1); + builder.lessThanEquals(node.column->column_id, node.column->predicate_type, + node.literals.front()); + return; + case OrcSargNodeKind::EQUALS: + DORIS_CHECK(node.column.has_value() && node.literals.size() == 1); + builder.equals(node.column->column_id, node.column->predicate_type, node.literals.front()); + return; + case OrcSargNodeKind::IN: + DORIS_CHECK(node.column.has_value() && node.literals.size() > 1); + builder.in(node.column->column_id, node.column->predicate_type, node.literals); + return; + case OrcSargNodeKind::IS_NULL: + DORIS_CHECK(node.column.has_value() && node.literals.empty()); + builder.isNull(node.column->column_id, node.column->predicate_type); + return; } } } // namespace bool build_orc_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, - const cctz::time_zone& timezone, const VExprSPtr& expr, + const cctz::time_zone& timezone, const OrcSargBuildOptions& options, + const VExprSPtr& expr, std::unique_ptr<::orc::SearchArgumentBuilder>& builder) { - return build_search_argument(request, root_type, timezone, expr, builder); + auto node = OrcSargCompiler(request, root_type, timezone, options).compile(expr); + if (!node.has_value()) { + return false; + } + emit_search_argument(*node, *builder); + return true; } } // namespace doris::format::orc diff --git a/be/src/format_v2/orc/orc_search_argument.h b/be/src/format_v2/orc/orc_search_argument.h index ad922915e6ed50..8cf3e23ac2aacf 100644 --- a/be/src/format_v2/orc/orc_search_argument.h +++ b/be/src/format_v2/orc/orc_search_argument.h @@ -31,11 +31,16 @@ class Type; namespace doris::format::orc { +struct OrcSargBuildOptions { + int max_pushdown_conditions_per_column = 1024; +}; + // Lower already-localized Doris file filters to ORC SearchArgument predicates. // TableColumnMapper owns table-schema -> file-local localization; this module // owns the ORC-specific type-id/literal lowering needed by the ORC C++ library. bool build_orc_search_argument(const format::FileScanRequest& request, const ::orc::Type& root_type, - const cctz::time_zone& timezone, const VExprSPtr& expr, + const cctz::time_zone& timezone, const OrcSargBuildOptions& options, + const VExprSPtr& expr, std::unique_ptr<::orc::SearchArgumentBuilder>& builder); } // namespace doris::format::orc diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp new file mode 100644 index 00000000000000..9eba6d8feaa55a --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -0,0 +1,874 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_desc.h" + +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/exception.h" +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "util/slice.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +static bool is_group_node(const tparquet::SchemaElement& schema) { + return schema.num_children > 0; +} + +static bool is_list_node(const tparquet::SchemaElement& schema) { + // Writers may emit only the modern logical type, so collection shape must not depend on the + // deprecated converted_type field being duplicated in the footer. + return (schema.__isset.converted_type && + schema.converted_type == tparquet::ConvertedType::LIST) || + (schema.__isset.logicalType && schema.logicalType.__isset.LIST); +} + +static bool is_map_node(const tparquet::SchemaElement& schema) { + return (schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::MAP || + schema.converted_type == tparquet::ConvertedType::MAP_KEY_VALUE)) || + (schema.__isset.logicalType && schema.logicalType.__isset.MAP); +} + +static bool has_primitive_only_annotation(const tparquet::SchemaElement& schema) { + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.DECIMAL || + logical.__isset.DATE || logical.__isset.TIME || logical.__isset.TIMESTAMP || + logical.__isset.INTEGER || logical.__isset.UNKNOWN || logical.__isset.JSON || + logical.__isset.BSON || logical.__isset.UUID || logical.__isset.FLOAT16 || + logical.__isset.GEOMETRY || logical.__isset.GEOGRAPHY) { + return true; + } + } + if (!schema.__isset.converted_type) { + return false; + } + return schema.converted_type != tparquet::ConvertedType::MAP && + schema.converted_type != tparquet::ConvertedType::MAP_KEY_VALUE && + schema.converted_type != tparquet::ConvertedType::LIST; +} + +static bool is_repeated_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REPEATED; +} + +static bool is_required_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REQUIRED; +} + +static bool is_optional_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::OPTIONAL; +} + +static int num_children_node(const tparquet::SchemaElement& schema) { + return schema.__isset.num_children ? schema.num_children : 0; +} + +static Status validate_native_schema_structure( + const std::vector& schemas) { + if (schemas.empty()) { + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + const auto& root = schemas[0]; + if (root.__isset.type || !root.__isset.num_children || root.num_children < 0 || + (root.__isset.repetition_type && + root.repetition_type != tparquet::FieldRepetitionType::REQUIRED)) { + // Writers may encode the root's implicit REQUIRED repetition explicitly, and a zero-child + // root is a valid metadata-only schema. Optional or repeated roots remain ambiguous. + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + struct PendingGroup { + size_t remaining_children; + size_t depth; + }; + const auto root_children = num_children_node(schemas[0]); + if (root_children < 0 || static_cast(root_children) > schemas.size() - 1) { + return Status::InvalidArgument("Invalid parquet root child count {}", root_children); + } + std::vector pending {{static_cast(root_children), 0}}; + for (size_t pos = 1; pos < schemas.size(); ++pos) { + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (pending.empty()) { + return Status::InvalidArgument("Schema element {} is not reachable from the root", pos); + } + + const size_t depth = pending.back().depth + 1; + --pending.back().remaining_children; + const auto& schema = schemas[pos]; + if (!schema.__isset.repetition_type) { + return Status::InvalidArgument("Schema element {} has no repetition type", pos); + } + const bool has_children = schema.__isset.num_children && schema.num_children > 0; + if (schema.__isset.type == has_children) { + // Some legacy parquet-cpp files explicitly encode num_children=0 on primitive nodes. + // Only a positive count denotes a group, preserving strict rejection of real dual-kind + // nodes without dropping those otherwise valid files. + return Status::InvalidArgument("Schema element {} has ambiguous primitive/group kind", + pos); + } + if (schema.__isset.num_children && schema.num_children < 0) { + return Status::InvalidArgument("Schema element {} has a negative child count", pos); + } + const int children = num_children_node(schemas[pos]); + if (children < 0 || static_cast(children) > schemas.size() - pos - 1) { + return Status::InvalidArgument("Invalid child count {} at schema element {}", children, + pos); + } + if (children > 0) { + // Bound the tree before any resize or recursion so an untrusted footer cannot + // turn a tiny schema into an unbounded allocation or parser stack. + if (depth > MAX_NATIVE_SCHEMA_DEPTH) { + return Status::InvalidArgument("Parquet schema depth {} exceeds limit {}", depth, + MAX_NATIVE_SCHEMA_DEPTH); + } + pending.push_back({static_cast(children), depth}); + } + } + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (!pending.empty()) { + return Status::InvalidArgument("Parquet schema ended before all children were parsed"); + } + return Status::OK(); +} + +/** + * `repeated_parent_def_level` is the definition level of the first ancestor node whose repetition_type equals REPEATED. + * Empty array/map values are not stored in doris columns, so have to use `repeated_parent_def_level` to skip the + * empty or null values in ancestor node. + * + * For instance, considering an array of strings with 3 rows like the following: + * null, [], [a, b, c] + * We can store four elements in data column: null, a, b, c + * and the offsets column is: 1, 1, 4 + * and the null map is: 1, 0, 0 + * For the i-th row in array column: range from `offsets[i - 1]` until `offsets[i]` represents the elements in this row, + * so we can't store empty array/map values in doris data column. + * As a comparison, spark does not require `repeated_parent_def_level`, + * because the spark column stores empty array/map values , and use anther length column to indicate empty values. + * Please reference: https://github.com/apache/spark/blob/master/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetColumnVector.java + * + * Furthermore, we can also avoid store null array/map values in doris data column. + * The same three rows as above, We can only store three elements in data column: a, b, c + * and the offsets column is: 0, 0, 3 + * and the null map is: 1, 0, 0 + * + * Inherit the repetition and definition level from parent node, if the parent node is repeated, + * we should set repeated_parent_def_level = definition_level, otherwise as repeated_parent_def_level. + * @param parent parent node + * @param repeated_parent_def_level the first ancestor node whose repetition_type equals REPEATED + */ +static void set_child_node_level(NativeFieldSchema* parent, int16_t repeated_parent_def_level) { + for (auto& child : parent->children) { + child.repetition_level = parent->repetition_level; + child.definition_level = parent->definition_level; + child.repeated_parent_def_level = repeated_parent_def_level; + } +} + +static bool is_struct_list_node(const tparquet::SchemaElement& schema, + const std::string& enclosing_list_name) { + // The legacy Parquet exception is exact: accepting every "*_tuple" wrapper changes a standard + // one-child LIST wrapper from ARRAY to ARRAY>. + return schema.name == "array" || schema.name == enclosing_list_name + "_tuple"; +} + +std::string NativeFieldSchema::debug_string() const { + std::stringstream ss; + ss << "NativeFieldSchema(name=" << name << ", R=" << repetition_level + << ", D=" << definition_level; + if (children.size() > 0) { + ss << ", type=" << data_type->get_name() << ", children=["; + for (int i = 0; i < children.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << children[i].debug_string(); + } + ss << "]"; + } else { + ss << ", physical_type=" << physical_type; + ss << " , doris_type=" << data_type->get_name(); + } + ss << ")"; + return ss.str(); +} + +Status NativeFieldDescriptor::parse_from_thrift( + const std::vector& t_schemas) { + _fields.clear(); + _physical_fields.clear(); + _name_to_field.clear(); + RETURN_IF_ERROR(validate_native_schema_structure(t_schemas)); + const auto& root_schema = t_schemas[0]; + _fields.resize(root_schema.num_children); + _next_schema_pos = 1; + + for (int i = 0; i < root_schema.num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &_fields[i])); + if (_name_to_field.find(_fields[i].name) != _name_to_field.end()) { + return Status::InvalidArgument("Duplicated field name: {}", _fields[i].name); + } + _name_to_field.emplace(_fields[i].name, &_fields[i]); + } + + if (_next_schema_pos != t_schemas.size()) { + return Status::InvalidArgument("Remaining {} unparsed schema elements", + t_schemas.size() - _next_schema_pos); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_node_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field) { + if (curr_pos >= t_schemas.size()) { + return Status::InvalidArgument("Out-of-bounds index of schema elements"); + } + auto& t_schema = t_schemas[curr_pos]; + if (is_group_node(t_schema)) { + // nested structure or nullable list + return parse_group_field(t_schemas, curr_pos, node_field); + } + if (is_repeated_node(t_schema)) { + // repeated (LIST) + // produce required list + node_field->repetition_level++; + node_field->definition_level++; + node_field->children.resize(1); + set_child_node_level(node_field, node_field->definition_level); + auto child = &node_field->children[0]; + parse_physical_field(t_schema, false, child); + + node_field->name = t_schema.name; + node_field->lower_case_name = to_lower(t_schema.name); + node_field->data_type = std::make_shared(make_nullable(child->data_type)); + _next_schema_pos = curr_pos + 1; + node_field->field_id = t_schema.__isset.field_id ? t_schema.field_id : -1; + } else { + bool is_optional = is_optional_node(t_schema); + if (is_optional) { + node_field->definition_level++; + } + parse_physical_field(t_schema, is_optional, node_field); + _next_schema_pos = curr_pos + 1; + } + return Status::OK(); +} + +void NativeFieldDescriptor::parse_physical_field(const tparquet::SchemaElement& physical_schema, + bool is_nullable, + NativeFieldSchema* physical_field) { + physical_field->name = physical_schema.name; + physical_field->lower_case_name = to_lower(physical_field->name); + physical_field->parquet_schema = physical_schema; + physical_field->physical_type = physical_schema.type; + physical_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + _physical_fields.push_back(physical_field); + physical_field->physical_column_index = cast_set(_physical_fields.size() - 1); + std::pair type; + try { + type = get_doris_type(physical_schema, is_nullable); + } catch (const Exception& e) { + const bool is_decimal = + (physical_schema.__isset.logicalType && + physical_schema.logicalType.__isset.DECIMAL) || + (physical_schema.__isset.converted_type && + physical_schema.converted_type == tparquet::ConvertedType::DECIMAL); + // DECIMAL conversion failures carry precision/scale semantics that must not be erased by + // a raw-byte fallback. Other unknown annotations intentionally retain the established + // physical fallback, including legacy INTERVAL footers and forward-compatible types. + if (is_decimal) { + physical_field->unsupported_reason = e.what(); + } + auto fallback_schema = physical_schema; + fallback_schema.__isset.logicalType = false; + fallback_schema.__isset.converted_type = false; + type = get_doris_type(fallback_schema, is_nullable); + } + physical_field->data_type = type.first; + physical_field->is_type_compatibility = type.second; + physical_field->field_id = physical_schema.__isset.field_id ? physical_schema.field_id : -1; +} + +std::pair NativeFieldDescriptor::get_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + if (physical_schema.__isset.logicalType) { + ans = convert_to_doris_type(physical_schema.logicalType, nullable); + } else if (physical_schema.__isset.converted_type) { + ans = convert_to_doris_type(physical_schema, nullable); + } + if (ans.first->get_primitive_type() == PrimitiveType::INVALID_TYPE) { + switch (physical_schema.type) { + case tparquet::Type::BOOLEAN: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BOOLEAN, nullable); + break; + case tparquet::Type::INT32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::Type::INT64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::Type::INT96: + if (_enable_mapping_timestamp_tz) { + // treat INT96 as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMPTZ, nullable, + 0, 6); + } else { + // in most cases, it's a nano timestamp + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, + 0, 6); + } + break; + case tparquet::Type::FLOAT: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + break; + case tparquet::Type::DOUBLE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, nullable); + break; + case tparquet::Type::BYTE_ARRAY: + if (_enable_mapping_varbinary) { + // if physical_schema not set logicalType and converted_type, + // we treat BYTE_ARRAY as VARBINARY by default, so that we can read all data directly. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet logicalType{}", + physical_schema.type)); + break; + } + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + tparquet::LogicalType logicalType, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + if (logicalType.__isset.STRING || logicalType.__isset.ENUM || logicalType.__isset.JSON || + logicalType.__isset.BSON) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } else if (logicalType.__isset.DECIMAL) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL128I, nullable, + logicalType.DECIMAL.precision, + logicalType.DECIMAL.scale); + } else if (logicalType.__isset.DATE) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + } else if (logicalType.__isset.INTEGER) { + if (logicalType.INTEGER.isSigned) { + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } + } else { + is_type_compatibility = true; + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + } + } + } else if (logicalType.__isset.TIME) { + const int scale = logicalType.TIME.unit.__isset.MILLIS ? 3 : 6; + // TIME stores an integer unit, so its Doris scale must preserve the footer unit or + // sub-second values are silently truncated by the target SerDe. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, scale); + } else if (logicalType.__isset.TIMESTAMP) { + if (_enable_mapping_timestamp_tz) { + if (logicalType.TIMESTAMP.isAdjustedToUTC) { + // treat TIMESTAMP with isAdjustedToUTC as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_TIMESTAMPTZ, nullable, 0, + logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + return ans; + } + } + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + } else if (logicalType.__isset.UUID) { + if (_enable_mapping_varbinary) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable, -1, + -1, 16); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + } else if (logicalType.__isset.FLOAT16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + } else { + throw Exception(Status::InternalError("Not supported parquet logicalType")); + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + switch (physical_schema.converted_type) { + case tparquet::ConvertedType::type::UTF8: + case tparquet::ConvertedType::type::ENUM: + case tparquet::ConvertedType::type::JSON: + case tparquet::ConvertedType::type::BSON: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + case tparquet::ConvertedType::type::DECIMAL: + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DECIMAL128I, nullable, physical_schema.precision, physical_schema.scale); + break; + case tparquet::ConvertedType::type::DATE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + break; + case tparquet::ConvertedType::type::TIME_MILLIS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 3); + break; + case tparquet::ConvertedType::type::TIME_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 6); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MILLIS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 3); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 6); + break; + case tparquet::ConvertedType::type::INT_8: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_8: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_16: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_16: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::ConvertedType::type::UINT_32: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_64: + is_type_compatibility = true; + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet ConvertedType: {}", + physical_schema.converted_type)); + } + return ans; +} + +Status NativeFieldDescriptor::parse_group_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field) { + auto& group_schema = t_schemas[curr_pos]; + if ((group_schema.__isset.logicalType && group_schema.logicalType.__isset.ENUM) || + (group_schema.__isset.converted_type && + group_schema.converted_type == tparquet::ConvertedType::ENUM)) { + // Preserve the established diagnostic used by compatibility tests and clients while the + // generic branch below handles every other primitive-only group annotation. + return Status::InvalidArgument("Logical type Enum cannot be applied to group node {}", + group_schema.name); + } + if (has_primitive_only_annotation(group_schema)) { + // Primitive annotations cannot change a group into a scalar. Reject them here instead of + // silently interpreting malformed metadata as an ordinary STRUCT. + return Status::InvalidArgument("Primitive annotation cannot be applied to group node {}", + group_schema.name); + } + if (is_map_node(group_schema)) { + // the map definition: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + return parse_map_field(t_schemas, curr_pos, group_field); + } + if (is_list_node(group_schema)) { + // the list definition: + // optional group (LIST) { + // repeated group [bag | list] { // hive or spark + // optional [array_element | element]; // hive or spark + // } + // } + return parse_list_field(t_schemas, curr_pos, group_field); + } + + if (is_repeated_node(group_schema)) { + group_field->repetition_level++; + group_field->definition_level++; + group_field->children.resize(1); + set_child_node_level(group_field, group_field->definition_level); + auto struct_field = &group_field->children[0]; + // the list of struct: + // repeated group (LIST) { + // optional/required ; + // ... + // } + // produce a non-null list + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, struct_field)); + + group_field->name = group_schema.name; + group_field->lower_case_name = to_lower(group_field->name); + group_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + group_field->data_type = + std::make_shared(make_nullable(struct_field->data_type)); + group_field->field_id = group_schema.__isset.field_id ? group_schema.field_id : -1; + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, group_field)); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_list_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field, bool repeated_node_is_enclosing_list_element) { + // the list definition: + // spark and hive have three level schemas but with different schema name + // spark: - "list" - "element" + // hive: - "bag" - "array_element" + // parse three level schemas to two level primitive like: LIST, + // or nested structure like: LIST> + auto& first_level = t_schemas[curr_pos]; + if (first_level.num_children != 1) { + return Status::InvalidArgument("List element should have only one child"); + } + + if (curr_pos + 1 >= t_schemas.size()) { + return Status::InvalidArgument("List element should have the second level schema"); + } + + if (first_level.repetition_type == tparquet::FieldRepetitionType::REPEATED && + !repeated_node_is_enclosing_list_element) { + return Status::InvalidArgument("List element can't be a repeated schema"); + } + + // the repeated schema element + auto& second_level = t_schemas[curr_pos + 1]; + if (second_level.repetition_type != tparquet::FieldRepetitionType::REPEATED) { + return Status::InvalidArgument("The second level of list element should be repeated"); + } + + // This indicates if this list is nullable. + bool is_optional = is_optional_node(first_level); + if (is_optional) { + list_field->definition_level++; + } + list_field->repetition_level++; + list_field->definition_level++; + list_field->children.resize(1); + NativeFieldSchema* list_child = &list_field->children[0]; + + size_t num_children = num_children_node(second_level); + if (num_children > 0) { + const bool structural_wrapper = is_struct_list_node(second_level, first_level.name); + const auto& only_child = t_schemas[curr_pos + 2]; + const bool single_key_set_wrapper = + second_level.__isset.converted_type && + second_level.converted_type == tparquet::ConvertedType::MAP_KEY_VALUE && + !is_repeated_node(only_child); + const bool nested_collection_annotation = + is_list_node(second_level) || + (is_map_node(second_level) && !single_key_set_wrapper); + if (num_children == 1 && !structural_wrapper && nested_collection_annotation) { + // The repeated node is already the outer LIST element. Preserve its own LIST/MAP + // annotation. A MAP_KEY_VALUE node with a repeated child is the legacy uncontained + // MAP shape; only its direct, non-repeated child denotes the enclosing SET key. + set_child_node_level(list_field, list_field->definition_level); + if (is_list_node(second_level)) { + RETURN_IF_ERROR(parse_list_field(t_schemas, curr_pos + 1, list_child, true)); + } else if (is_map_node(second_level)) { + RETURN_IF_ERROR(parse_map_field(t_schemas, curr_pos + 1, list_child, true)); + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 1 && !structural_wrapper && !is_repeated_node(only_child)) { + // optional field, and the third level element is the nested structure in list + // produce nested structure like: LIST, LIST, LIST> + // skip bag/list, it's a repeated element. + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_node_field(t_schemas, curr_pos + 2, list_child)); + } else { + // required field, produce the list of struct + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 0) { + // required two level list, for compatibility reason. + set_child_node_level(list_field, list_field->definition_level); + parse_physical_field(second_level, false, list_child); + _next_schema_pos = curr_pos + 2; + } + + list_field->name = first_level.name; + list_field->lower_case_name = to_lower(first_level.name); + list_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + list_field->data_type = + std::make_shared(make_nullable(list_field->children[0].data_type)); + if (is_optional) { + list_field->data_type = make_nullable(list_field->data_type); + } + list_field->field_id = first_level.__isset.field_id ? first_level.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_map_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element) { + // the map definition in parquet: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + // Map value can be optional, the map without values is a SET + if (curr_pos + 2 >= t_schemas.size()) { + return Status::InvalidArgument("Map element should have at least three levels"); + } + auto& map_schema = t_schemas[curr_pos]; + if (map_schema.num_children != 1) { + return Status::InvalidArgument( + "Map element should have only one child(name='map', type='MAP_KEY_VALUE')"); + } + if (is_repeated_node(map_schema) && !repeated_node_is_enclosing_list_element) { + return Status::InvalidArgument("Map element can't be a repeated schema"); + } + auto& map_key_value = t_schemas[curr_pos + 1]; + if (!is_group_node(map_key_value) || !is_repeated_node(map_key_value)) { + return Status::InvalidArgument( + "the second level in map must be a repeated group(key and value)"); + } + auto& map_key = t_schemas[curr_pos + 2]; + if (!is_required_node(map_key)) { + LOG(WARNING) << "Filed " << map_schema.name << " is map type, but with nullable key column"; + } + + if (map_key_value.num_children == 1) { + // The map with three levels is a SET + return parse_list_field(t_schemas, curr_pos, map_field, + repeated_node_is_enclosing_list_element); + } + if (map_key_value.num_children != 2) { + // A standard map should have four levels + return Status::InvalidArgument( + "the second level in map(MAP_KEY_VALUE) should have two children"); + } + // standard map + bool is_optional = is_optional_node(map_schema); + if (is_optional) { + map_field->definition_level++; + } + map_field->repetition_level++; + map_field->definition_level++; + + // Directly create key and value children instead of intermediate key_value node + map_field->children.resize(2); + // map is a repeated node, we should set the `repeated_parent_def_level` of its children as `definition_level` + set_child_node_level(map_field, map_field->definition_level); + + auto key_field = &map_field->children[0]; + auto value_field = &map_field->children[1]; + + // Parse key and value fields directly from the key_value group's children + _next_schema_pos = curr_pos + 2; // Skip key_value group, go directly to key + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, key_field)); + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, value_field)); + + map_field->name = map_schema.name; + map_field->lower_case_name = to_lower(map_field->name); + map_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + map_field->data_type = std::make_shared(make_nullable(key_field->data_type), + make_nullable(value_field->data_type)); + if (is_optional) { + map_field->data_type = make_nullable(map_field->data_type); + } + map_field->field_id = map_schema.__isset.field_id ? map_schema.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_struct_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* struct_field) { + // the nested column in parquet, parse group to struct. + auto& struct_schema = t_schemas[curr_pos]; + bool is_optional = is_optional_node(struct_schema); + if (is_optional) { + struct_field->definition_level++; + } + auto num_children = struct_schema.num_children; + struct_field->children.resize(num_children); + set_child_node_level(struct_field, struct_field->repeated_parent_def_level); + _next_schema_pos = curr_pos + 1; + for (int i = 0; i < num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &struct_field->children[i])); + } + struct_field->name = struct_schema.name; + struct_field->lower_case_name = to_lower(struct_field->name); + struct_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + + struct_field->field_id = struct_schema.__isset.field_id ? struct_schema.field_id : -1; + DataTypes res_data_types; + std::vector names; + for (int i = 0; i < num_children; ++i) { + res_data_types.push_back(make_nullable(struct_field->children[i].data_type)); + names.push_back(struct_field->children[i].name); + } + struct_field->data_type = std::make_shared(res_data_types, names); + if (is_optional) { + struct_field->data_type = make_nullable(struct_field->data_type); + } + return Status::OK(); +} + +int NativeFieldDescriptor::get_column_index(const std::string& column) const { + for (int32_t i = 0; i < _fields.size(); i++) { + if (_fields[i].name == column) { + return i; + } + } + return -1; +} + +NativeFieldSchema* NativeFieldDescriptor::get_column(const std::string& name) const { + auto it = _name_to_field.find(name); + if (it != _name_to_field.end()) { + return it->second; + } + throw Exception(Status::InternalError("Name {} not found in NativeFieldDescriptor!", name)); + return nullptr; +} + +void NativeFieldDescriptor::get_column_names(std::unordered_set* names) const { + names->clear(); + for (const NativeFieldSchema& f : _fields) { + names->emplace(f.name); + } +} + +std::string NativeFieldDescriptor::debug_string() const { + std::stringstream ss; + ss << "fields=["; + for (int i = 0; i < _fields.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << _fields[i].debug_string(); + } + ss << "]"; + return ss.str(); +} + +void NativeFieldDescriptor::assign_ids() { + uint64_t next_id = 1; + for (auto& field : _fields) { + field.assign_ids(next_id); + } +} + +const NativeFieldSchema* NativeFieldDescriptor::find_column_by_id(uint64_t column_id) const { + for (const auto& field : _fields) { + if (auto result = field.find_column_by_id(column_id)) { + return result; + } + } + return nullptr; +} + +void NativeFieldSchema::assign_ids(uint64_t& next_id) { + column_id = next_id++; + + for (auto& child : children) { + child.assign_ids(next_id); + } + + max_column_id = next_id - 1; +} + +const NativeFieldSchema* NativeFieldSchema::find_column_by_id(uint64_t target_id) const { + if (column_id == target_id) { + return this; + } + + for (const auto& child : children) { + if (auto result = child.find_column_by_id(target_id)) { + return result; + } + } + + return nullptr; +} + +uint64_t NativeFieldSchema::get_column_id() const { + return column_id; +} + +void NativeFieldSchema::set_column_id(uint64_t id) { + column_id = id; +} + +uint64_t NativeFieldSchema::get_max_column_id() const { + return max_column_id; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_desc.h b/be/src/format_v2/parquet/native_schema_desc.h new file mode 100644 index 00000000000000..dfb669559fc71a --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.h @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nothing.h" +#include "util/slice.h" + +namespace doris::format::parquet { + +// Constant for unassigned column IDs +constexpr uint64_t NATIVE_UNASSIGNED_COLUMN_ID = UINT64_MAX; +constexpr size_t MAX_NATIVE_SCHEMA_DEPTH = 100; + +struct NativeFieldSchema { + std::string name; + std::string lower_case_name; // for hms column name case insensitive match + // the referenced parquet schema element + tparquet::SchemaElement parquet_schema; + + // Used to identify whether this field is a nested field. + DataTypePtr data_type; + // Schema construction keeps a physical fallback so unprojected columns and metadata-only + // queries remain readable, while projection validation reports the original logical failure. + std::string unsupported_reason; + + // Only valid when this field is a leaf node + tparquet::Type::type physical_type; + // The index order in NativeFieldDescriptor._physical_fields + int physical_column_index = -1; + int16_t definition_level = 0; + int16_t repetition_level = 0; + int16_t repeated_parent_def_level = 0; + std::vector children; + + //For UInt8 -> Int16,UInt16 -> Int32,UInt32 -> Int64,UInt64 -> Int128. + bool is_type_compatibility = false; + + NativeFieldSchema() + : data_type(std::make_shared()), + column_id(NATIVE_UNASSIGNED_COLUMN_ID) {} + ~NativeFieldSchema() = default; + NativeFieldSchema(const NativeFieldSchema& fieldSchema) = default; + std::string debug_string() const; + + int32_t field_id = -1; + uint64_t column_id = NATIVE_UNASSIGNED_COLUMN_ID; + uint64_t max_column_id = 0; // Maximum column ID for this field and its children + + // Column ID assignment and lookup methods + void assign_ids(uint64_t& next_id); + const NativeFieldSchema* find_column_by_id(uint64_t target_id) const; + uint64_t get_column_id() const; + void set_column_id(uint64_t id); + uint64_t get_max_column_id() const; +}; + +// V2 owns this schema tree and parser so footer/schema planning never invokes the V1 reader path. +class NativeFieldDescriptor { +private: + // Only the schema elements at the first level + std::vector _fields; + // The leaf node of schema elements + std::vector _physical_fields; + // Name to _fields, not all schema elements + std::unordered_map _name_to_field; + // Used in from_thrift, marking the next schema position that should be parsed + size_t _next_schema_pos; + // useful for parse_node_field to decide whether to convert byte_array to VARBINARY type + bool _enable_mapping_varbinary = false; + bool _enable_mapping_timestamp_tz = false; + +private: + void parse_physical_field(const tparquet::SchemaElement& physical_schema, bool is_nullable, + NativeFieldSchema* physical_field); + + Status parse_list_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field, + bool repeated_node_is_enclosing_list_element = false); + + Status parse_map_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element = false); + + Status parse_struct_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* struct_field); + + Status parse_group_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field); + + Status parse_node_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field); + + std::pair convert_to_doris_type(tparquet::LogicalType logicalType, + bool nullable); + std::pair convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable); + std::pair get_doris_type(const tparquet::SchemaElement& physical_schema, + bool nullable); + +public: + NativeFieldDescriptor() = default; + ~NativeFieldDescriptor() = default; + + /** + * Parse NativeFieldDescriptor from parquet thrift FileMetaData. + * @param t_schemas list of schema elements + */ + Status parse_from_thrift(const std::vector& t_schemas); + + int get_column_index(const std::string& column) const; + + /** + * Get the column(the first level schema element, maybe nested field) by index. + * @param index Column index in _fields + */ + const NativeFieldSchema* get_column(size_t index) const { return &_fields[index]; } + + /** + * Get the column(the first level schema element, maybe nested field) by name. + * @param name Column name + * @return NativeFieldSchema or nullptr if not exists + */ + NativeFieldSchema* get_column(const std::string& name) const; + + void get_column_names(std::unordered_set* names) const; + + std::string debug_string() const; + + int32_t size() const { return cast_set(_fields.size()); } + + size_t physical_fields_size() const { return _physical_fields.size(); } + + const NativeFieldSchema* get_physical_field(size_t index) const { + return _physical_fields[index]; + } + + const std::vector& get_fields_schema() const { return _fields; } + + /** + * Assign stable column IDs to schema fields. + * + * This uses an ORC-compatible encoding so that the results of + * create_column_ids() are consistent across formats. IDs start from 1 + * and are assigned in a pre-order traversal (parent before children). + * After calling this, each NativeFieldSchema will have column_id and + * max_column_id populated. + */ + void assign_ids(); + + const NativeFieldSchema* find_column_by_id(uint64_t column_id) const; + void set_enable_mapping_varbinary(bool enable) { _enable_mapping_varbinary = enable; } + void set_enable_mapping_timestamp_tz(bool enable) { _enable_mapping_timestamp_tz = enable; } +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.cpp b/be/src/format_v2/parquet/native_schema_node.cpp new file mode 100644 index 00000000000000..052df93e4b4143 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.cpp @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_node.h" + +#include +#include + +#include "core/assert_cast.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +void NativeStructSchemaNode::add_child(std::string table_name, std::string file_name, + std::shared_ptr node) { + _children.emplace(std::move(table_name), Child {std::move(file_name), std::move(node)}); +} + +void NativeStructSchemaNode::add_missing_child(std::string table_name) { + _children.emplace(std::move(table_name), Child {}); +} + +std::shared_ptr NativeStructSchemaNode::child( + const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? nullptr : it->second.node; +} + +std::string NativeStructSchemaNode::file_child_name(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? std::string {} : it->second.file_name; +} + +bool NativeStructSchemaNode::has_child(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it != _children.end() && it->second.node != nullptr; +} + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result) { + if (projected_type == nullptr || result == nullptr) { + return Status::InvalidArgument("Native Parquet schema mapping input is null"); + } + const auto type = remove_nullable(projected_type); + switch (type->get_primitive_type()) { + case TYPE_STRUCT: { + if (file_schema.kind != ParquetColumnSchemaKind::STRUCT) { + return Status::Corruption("Parquet column {} is not a STRUCT", file_schema.name); + } + const auto* struct_type = assert_cast(type.get()); + auto node = std::make_shared(); + for (size_t i = 0; i < struct_type->get_elements().size(); ++i) { + const auto& table_name = struct_type->get_element_name(i); + const ParquetColumnSchema* file_child = nullptr; + for (const auto& child : file_schema.children) { + if (child->name == table_name) { + file_child = child.get(); + break; + } + } + if (file_child == nullptr) { + for (const auto& child : file_schema.children) { + if (to_lower(child->name) != to_lower(table_name)) { + continue; + } + if (UNLIKELY(file_child != nullptr)) { + // Exact writer identity is authoritative; normalized fallback is only safe + // when the requested field has one physical candidate. + return Status::Corruption( + "Parquet STRUCT {} has ambiguous case-insensitive child name {}", + file_schema.name, table_name); + } + file_child = child.get(); + } + } + if (file_child == nullptr) { + node->add_missing_child(table_name); + continue; + } + std::shared_ptr child_node; + RETURN_IF_ERROR(build_native_schema_node(struct_type->get_element(i), *file_child, + &child_node)); + node->add_child(table_name, file_child->name, std::move(child_node)); + } + *result = std::move(node); + return Status::OK(); + } + case TYPE_ARRAY: { + if (file_schema.kind != ParquetColumnSchemaKind::LIST || file_schema.children.size() != 1) { + return Status::Corruption("Parquet column {} is not an ARRAY", file_schema.name); + } + const auto* array_type = assert_cast(type.get()); + std::shared_ptr element; + RETURN_IF_ERROR(build_native_schema_node(array_type->get_nested_type(), + *file_schema.children[0], &element)); + *result = std::make_shared(std::move(element)); + return Status::OK(); + } + case TYPE_MAP: { + if (file_schema.kind != ParquetColumnSchemaKind::MAP || file_schema.children.size() != 2) { + return Status::Corruption("Parquet column {} is not a MAP", file_schema.name); + } + const auto* map_type = assert_cast(type.get()); + std::shared_ptr key; + std::shared_ptr value; + RETURN_IF_ERROR( + build_native_schema_node(map_type->get_key_type(), *file_schema.children[0], &key)); + RETURN_IF_ERROR(build_native_schema_node(map_type->get_value_type(), + *file_schema.children[1], &value)); + *result = std::make_shared(std::move(key), std::move(value)); + return Status::OK(); + } + default: + if (file_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { + return Status::Corruption("Parquet column {} is not a scalar", file_schema.name); + } + *result = std::make_shared(); + return Status::OK(); + } +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.h b/be/src/format_v2/parquet/native_schema_node.h new file mode 100644 index 00000000000000..25c42ad4b2c6d4 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" + +namespace doris::format::parquet { + +struct ParquetColumnSchema; + +// V2-owned semantic mapping consumed by the native decoder. It deliberately contains no V1 +// reader/schema-helper state, so FileScannerV2 can build the mapping from its native metadata tree. +class NativeSchemaNode { +public: + virtual ~NativeSchemaNode() = default; + + virtual std::shared_ptr child(const std::string&) const { return nullptr; } + virtual std::string file_child_name(const std::string&) const { return {}; } + virtual bool has_child(const std::string&) const { return false; } + virtual std::shared_ptr element() const { return nullptr; } + virtual std::shared_ptr key() const { return nullptr; } + virtual std::shared_ptr value() const { return nullptr; } +}; + +class NativeScalarSchemaNode final : public NativeSchemaNode {}; + +class NativeStructSchemaNode final : public NativeSchemaNode { +public: + void add_child(std::string table_name, std::string file_name, + std::shared_ptr node); + void add_missing_child(std::string table_name); + + std::shared_ptr child(const std::string& table_name) const override; + std::string file_child_name(const std::string& table_name) const override; + bool has_child(const std::string& table_name) const override; + +private: + struct Child { + std::string file_name; + std::shared_ptr node; + }; + std::map _children; +}; + +class NativeArraySchemaNode final : public NativeSchemaNode { +public: + explicit NativeArraySchemaNode(std::shared_ptr element) + : _element(std::move(element)) {} + std::shared_ptr element() const override { return _element; } + +private: + std::shared_ptr _element; +}; + +class NativeMapSchemaNode final : public NativeSchemaNode { +public: + NativeMapSchemaNode(std::shared_ptr key, + std::shared_ptr value) + : _key(std::move(key)), _value(std::move(value)) {} + std::shared_ptr key() const override { return _key; } + std::shared_ptr value() const override { return _value; } + +private: + std::shared_ptr _key; + std::shared_ptr _value; +}; + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp index b42d47987a54cb..7e541b57da2780 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.cpp +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -15,476 +15,215 @@ #include "format_v2/parquet/parquet_column_schema.h" -#include - #include #include #include #include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/parquet_type.h" namespace doris::format::parquet { namespace { -struct SchemaBuildContext { - int32_t local_id = -1; // child ordinal in the parent node - int16_t definition_level = 0; // accumulated optional/repeated level count - int16_t repetition_level = 0; // accumulated repeated level count - int16_t nullable_definition_level = 0; // definition level of the nearest optional node - int16_t repeated_repetition_level = 0; // repetition level of the nearest repeated node - int16_t repeated_ancestor_definition_level = 0; // definition level of the nearest repeated node -}; - -enum class SchemaBuildMode { - // Normal recursive schema build. Bare repeated fields are exposed as Doris ARRAY for - // protobuf/legacy Parquet compatibility, while repeated LIST/MAP annotated groups are rejected - // because Parquet LIST/MAP outer groups are not allowed to be repeated at a top-level or struct - // field boundary. - NORMAL, - // Build the current repeated node as the already-selected element of an enclosing LIST. This - // is the compatibility path for Arrow/parquet-format legacy two-level LIST encodings where the - // repeated node itself is the array element instead of a wrapper that should be stripped. - REPEATED_NODE_AS_LIST_ELEMENT, - // Build the current repeated group as a STRUCT element of an enclosing LIST, ignoring LIST/MAP - // annotations on the repeated group itself. This keeps compatibility with the old Doris - // Parquet schema parser for Hive/legacy wrappers named "array" or "_tuple". - REPEATED_NODE_AS_STRUCT_ELEMENT, -}; - -// Result of applying Parquet LIST backward compatibility rules to the single repeated child of a -// LIST-annotated group. The repeated child can either be a physical wrapper whose only child is the -// element, or the element node itself. -struct ListElementResolution { - // Parquet node that should be exposed as Doris ARRAY element. - const ::parquet::schema::Node* element_node = nullptr; - // Level state after consuming the LIST repeated child. The parent ARRAY schema keeps this state - // to materialize offsets, empty arrays and null arrays. - SchemaBuildContext repeated_context; - // Level state used to build element_node. This equals repeated_context when the repeated child - // itself is the element, and includes the wrapper's only child when standard 3-level LIST - // encoding is stripped. - SchemaBuildContext element_context; - // Build mode for element_node. Non-NORMAL modes mean element_node is the repeated child itself, - // and the repeated level must not be interpreted as a second unrelated array at the same - // boundary. - SchemaBuildMode element_build_mode = SchemaBuildMode::NORMAL; -}; - -// Resolved repeated entry group of a MAP-annotated group. The entry wrapper is a physical Parquet -// encoding detail; Doris folds it into the parent MAP schema and exposes only direct [key, value] -// children. -struct MapEntryResolution { - const ::parquet::schema::GroupNode* entry_group = nullptr; - // Level state after consuming the repeated entry group. The parent MAP schema keeps this state - // to materialize offsets, empty maps and null maps. - SchemaBuildContext entry_context; -}; - -bool is_list_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::LIST || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_list()); -} - -bool is_map_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::MAP || - node.converted_type() == ::parquet::ConvertedType::MAP_KEY_VALUE || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_map()); -} - -bool has_logical_annotation(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return (node.converted_type() != ::parquet::ConvertedType::NONE && - node.converted_type() != ::parquet::ConvertedType::UNDEFINED) || - (logical_type != nullptr && logical_type->is_valid() && !logical_type->is_none()); -} - -bool has_structural_list_name(const std::string& list_name, const std::string& repeated_name) { - return repeated_name == "array" || repeated_name == list_name + "_tuple"; -} - -bool should_build_repeated_field_as_list(const ::parquet::schema::Node& node) { - return node.is_repeated() && !is_list_node(node) && !is_map_node(node); -} - -DataTypePtr nullable_if_needed(DataTypePtr type, const ::parquet::schema::Node& node) { - return node.is_optional() ? make_nullable(type) : type; -} - -void inherit_common_schema_state(const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - column_schema->local_id = context.local_id; - column_schema->parquet_field_id = node.field_id(); - column_schema->name = node.name(); - column_schema->max_definition_level = context.definition_level; - column_schema->max_repetition_level = context.repetition_level; - column_schema->nullable_definition_level = context.nullable_definition_level; - column_schema->definition_level = context.definition_level; - column_schema->repetition_level = context.repetition_level; - column_schema->repeated_ancestor_definition_level = context.repeated_ancestor_definition_level; - column_schema->repeated_repetition_level = context.repeated_repetition_level; -} - -SchemaBuildContext child_context(const SchemaBuildContext& parent, - const ::parquet::schema::Node& child_node, int32_t child_idx) { - SchemaBuildContext result = parent; - result.local_id = child_idx; - if (child_node.repetition() == ::parquet::Repetition::OPTIONAL) { - result.definition_level++; - result.nullable_definition_level = result.definition_level; +ParquetTimeUnit native_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; } - if (child_node.is_repeated()) { - result.repetition_level++; - result.definition_level++; - result.repeated_repetition_level = result.repetition_level; - result.repeated_ancestor_definition_level = result.definition_level; + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; } - return result; -} - -void propagate_child_levels(ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - for (const auto& child : column_schema->children) { - column_schema->max_definition_level = - std::max(column_schema->max_definition_level, child->max_definition_level); - column_schema->max_repetition_level = - std::max(column_schema->max_repetition_level, child->max_repetition_level); + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; } + return ParquetTimeUnit::UNKNOWN; } -// Mirrors Arrow's ResolveList() compatibility rules, but only decides which Parquet node is the -// logical LIST element. The caller still builds Doris' semantic LIST->[element] schema tree. -// Important cases: -// - repeated primitive: the primitive itself is the element (legacy two-level LIST). -// - repeated group with multiple children: the group itself is a STRUCT element. -// - repeated group named "array" or "_tuple": the group itself is a STRUCT element per -// Parquet backward compatibility rules, even when it has one child or its own logical annotation. -// This also keeps v2 file-local schema aligned with Doris' old schema parser used by HDFS TVF. -// - other repeated group with a logical annotation, or whose only child is repeated: the group -// itself is the element. This preserves nested LIST/MAP and repeated fields inside struct -// elements. -// - otherwise, strip the one-child repeated wrapper as standard three-level LIST encoding. -Status resolve_list_element_node(const ::parquet::schema::GroupNode& list_group, - const SchemaBuildContext& list_context, - ListElementResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (list_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - const auto& repeated_node = *list_group.field(0); - if (!repeated_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - result->repeated_context = child_context(list_context, repeated_node, 0); - if (repeated_node.is_primitive()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - const auto& repeated_group = static_cast(repeated_node); - if (repeated_group.field_count() == 0) { - return Status::NotSupported("Unsupported parquet LIST element layout for column {}", - list_group.name()); - } - const bool repeated_group_has_logical_annotation = has_logical_annotation(repeated_group); - if (repeated_group.field_count() > 1 || - has_structural_list_name(list_group.name(), repeated_group.name())) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT; - return Status::OK(); - } - if (repeated_group_has_logical_annotation) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - const auto& only_child = *repeated_group.field(0); - if (only_child.is_repeated()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); +ParquetExtraTypeInfo native_time_extra(ParquetTimeUnit unit) { + switch (unit) { + case ParquetTimeUnit::MILLIS: + return ParquetExtraTypeInfo::UNIT_MS; + case ParquetTimeUnit::MICROS: + return ParquetExtraTypeInfo::UNIT_MICROS; + case ParquetTimeUnit::NANOS: + return ParquetExtraTypeInfo::UNIT_NS; + case ParquetTimeUnit::UNKNOWN: + default: + return ParquetExtraTypeInfo::NONE; } - - result->element_node = &only_child; - result->element_context = child_context(result->repeated_context, only_child, 0); - return Status::OK(); -} - -// Resolves the repeated entry group of a MAP/MAP_KEY_VALUE node. Unlike LIST, MAP has no supported -// two-level form in this reader: Doris requires a repeated group with exactly key and value -// children, then folds that physical entry group out of ParquetColumnSchema. Some external writers -// emit optional MAP keys even though standard Parquet MAP keys are required; keep the key's -// definition levels and expose it as nullable for compatibility with the old reader. -Status resolve_map_entry_group(const ::parquet::schema::GroupNode& map_group, - const SchemaBuildContext& map_context, MapEntryResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (map_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - const auto& entry_node = *map_group.field(0); - if (!entry_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - if (entry_node.is_primitive()) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - const auto& entry_group = static_cast(entry_node); - if (entry_group.field_count() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - // The Parquet logical MAP spec requires key to be REQUIRED. Some legacy/Hive-written files - // still mark the key field OPTIONAL even when all actual keys are non-null, for example: - // optional group t_map_varchar (MAP) { - // repeated group key_value { - // optional binary key (STRING); - // optional binary value (STRING); - // } - // } - // Accept that schema here so compatible files can be read. MapColumnReader validates the - // materialized key column and rejects data that really contains null map keys. - result->entry_group = &entry_group; - result->entry_context = child_context(map_context, entry_node, 0); - return Status::OK(); -} - -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode); - -// Builds a semantic ARRAY schema for a bare repeated field. Arrow handles this in -// NodeToSchemaField()/GroupToSchemaField(); Doris needs the same compatibility behavior because -// protobuf and old parquet writers often encode repeated fields without a LIST annotation. -// Example: -// optional group event { -// repeated group links { -// optional binary url (UTF8); -// optional int32 rank; -// } -// } -// Doris exposes event.links as ARRAY>, not STRUCT. This keeps v2's -// file-local schema aligned with the old schema parser used by HDFS TVF schema fetching. -// When the repeated field appears inside an already resolved LIST element, only the nested repeated -// child should be wrapped: -// optional group a (LIST) { -// repeated group element { -// repeated int32 items; -// } -// } -// The outer LIST element is the repeated "element" group, and its repeated "items" child should be -// represented as a field of type ARRAY inside the struct element. -Status build_repeated_field_as_list_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& repeated_node, - const SchemaBuildContext& repeated_context, - std::unique_ptr* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - auto list_schema = std::make_unique(); - inherit_common_schema_state(repeated_node, repeated_context, list_schema.get()); - list_schema->kind = ParquetColumnSchemaKind::LIST; - list_schema->definition_level = repeated_context.definition_level; - list_schema->repetition_level = repeated_context.repetition_level; - list_schema->repeated_repetition_level = repeated_context.repeated_repetition_level; - - std::unique_ptr element_child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, repeated_node, repeated_context, - &element_child, - SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT)); - element_child->name = "element"; - list_schema->type = std::make_shared(element_child->type); - list_schema->children.push_back(std::move(element_child)); - propagate_child_levels(list_schema.get()); - *result = std::move(list_schema); - return Status::OK(); } -// Recursively builds ParquetColumnSchema for the given schema node and its children in Parquet -// file's metadata. NORMAL mode exposes bare repeated fields as ARRAY for legacy compatibility. -// REPEATED_NODE_AS_LIST_ELEMENT mode means the current repeated node was already selected as an -// enclosing LIST element, so only its nested bare repeated children should be wrapped. -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (mode == SchemaBuildMode::NORMAL && should_build_repeated_field_as_list(node)) { - return build_repeated_field_as_list_schema(schema, node, context, result); - } - - auto column_schema = std::make_unique(); - inherit_common_schema_state(node, context, column_schema.get()); - - if (node.is_primitive()) { - const int leaf_column_id = schema.ColumnIndex(node); - if (leaf_column_id < 0) { - return Status::InvalidArgument("Cannot find leaf column id for parquet column {}", - node.name()); - } - column_schema->kind = ParquetColumnSchemaKind::PRIMITIVE; - column_schema->leaf_column_id = leaf_column_id; - column_schema->descriptor = schema.Column(leaf_column_id); - if (column_schema->descriptor != nullptr) { - column_schema->max_definition_level = column_schema->descriptor->max_definition_level(); - column_schema->max_repetition_level = column_schema->descriptor->max_repetition_level(); - } - column_schema->type_descriptor = resolve_parquet_type(column_schema->descriptor); - column_schema->type = column_schema->type_descriptor.doris_type; - if (column_schema->type == nullptr) { - if (!column_schema->type_descriptor.unsupported_reason.empty()) { - return Status::NotSupported("Unsupported parquet column '{}': {}", node.name(), - column_schema->type_descriptor.unsupported_reason); +void fill_native_type_descriptor(const NativeFieldSchema& field, ParquetTypeDescriptor* result) { + DORIS_CHECK(result != nullptr); + const auto& schema = field.parquet_schema; + result->doris_type = field.data_type; + result->unsupported_reason = field.unsupported_reason; + result->physical_type = static_cast(field.physical_type); + result->fixed_length = schema.__isset.type_length ? schema.type_length : -1; + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.DECIMAL) { + result->is_decimal = true; + result->decimal_precision = logical.DECIMAL.precision; + result->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.INTEGER) { + result->integer_bit_width = logical.INTEGER.bitWidth; + result->is_unsigned_integer = !logical.INTEGER.isSigned; + } else if (logical.__isset.TIME) { + result->time_unit = native_time_unit(logical.TIME.unit); + result->extra_type_info = native_time_extra(result->time_unit); + if (logical.TIME.isAdjustedToUTC) { + result->unsupported_reason = + "Parquet TIME with isAdjustedToUTC=true is not supported"; } - return Status::NotSupported("Unsupported parquet column type for column {}", - node.name()); + } else if (logical.__isset.TIMESTAMP) { + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + result->time_unit = native_time_unit(logical.TIMESTAMP.unit); + result->extra_type_info = native_time_extra(result->time_unit); + } else if (logical.__isset.FLOAT16) { + result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; } - column_schema->type = node.is_optional() - ? make_nullable(remove_nullable(column_schema->type)) - : remove_nullable(column_schema->type); - *result = std::move(column_schema); - return Status::OK(); - } - - const auto& group = static_cast(node); - if (is_list_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet LIST column {}", node.name()); + } else if (schema.__isset.converted_type) { + switch (schema.converted_type) { + case tparquet::ConvertedType::DECIMAL: + result->is_decimal = true; + result->decimal_precision = schema.__isset.precision ? schema.precision : -1; + result->decimal_scale = schema.__isset.scale ? schema.scale : -1; + break; + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::UINT_8: + result->integer_bit_width = 8; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_8; + break; + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::UINT_16: + result->integer_bit_width = 16; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_16; + break; + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::UINT_32: + result->integer_bit_width = 32; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_32; + break; + case tparquet::ConvertedType::INT_64: + case tparquet::ConvertedType::UINT_64: + result->integer_bit_width = 64; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_64; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = true; + result->time_unit = schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS + ? ParquetTimeUnit::MILLIS + : ParquetTimeUnit::MICROS; + result->extra_type_info = native_time_extra(result->time_unit); + break; + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::TIME_MICROS: + result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; + break; + default: + break; } - column_schema->kind = ParquetColumnSchemaKind::LIST; - ListElementResolution list_element; - RETURN_IF_ERROR(resolve_list_element_node(group, context, &list_element)); - column_schema->definition_level = list_element.repeated_context.definition_level; - column_schema->repetition_level = list_element.repeated_context.repetition_level; - column_schema->repeated_repetition_level = - list_element.repeated_context.repeated_repetition_level; - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, *list_element.element_node, - list_element.element_context, &child, - list_element.element_build_mode)); - child->name = "element"; - column_schema->type = - nullable_if_needed(std::make_shared(child->type), node); - column_schema->children.push_back(std::move(child)); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); } - if (is_map_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet MAP column {}", node.name()); - } - column_schema->kind = ParquetColumnSchemaKind::MAP; - MapEntryResolution map_entry; - RETURN_IF_ERROR(resolve_map_entry_group(group, context, &map_entry)); - column_schema->definition_level = map_entry.entry_context.definition_level; - column_schema->repetition_level = map_entry.entry_context.repetition_level; - column_schema->repeated_repetition_level = - map_entry.entry_context.repeated_repetition_level; - for (int child_idx = 0; child_idx < map_entry.entry_group->field_count(); ++child_idx) { - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode( - schema, *map_entry.entry_group->field(child_idx), - child_context(map_entry.entry_context, *map_entry.entry_group->field(child_idx), - child_idx), - &child, SchemaBuildMode::NORMAL)); - child->name = child_idx == 0 ? "key" : "value"; - column_schema->children.push_back(std::move(child)); + if (result->is_decimal) { + switch (result->physical_type) { + case tparquet::Type::INT32: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; + break; + case tparquet::Type::INT64: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; + break; + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; + break; + default: + break; } - if (column_schema->children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - node.name()); - } - auto key_type = make_nullable(column_schema->children[0]->type); - auto value_type = make_nullable(column_schema->children[1]->type); - column_schema->type = - nullable_if_needed(std::make_shared(key_type, value_type), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); - } + } else if (result->physical_type == tparquet::Type::INT96) { + result->is_timestamp = true; + result->extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; + } + result->is_string_like = !result->is_decimal && + result->extra_type_info != ParquetExtraTypeInfo::FLOAT16 && + (result->physical_type == tparquet::Type::BYTE_ARRAY || + result->physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY); +} - column_schema->kind = ParquetColumnSchemaKind::STRUCT; - DataTypes child_types; - Strings child_names; - child_types.reserve(group.field_count()); - child_names.reserve(group.field_count()); - for (int child_idx = 0; child_idx < group.field_count(); ++child_idx) { - const auto& child_node = *group.field(child_idx); - std::unique_ptr child; - const auto child_ctx = child_context(context, child_node, child_idx); - if (should_build_repeated_field_as_list(child_node)) { - RETURN_IF_ERROR( - build_repeated_field_as_list_schema(schema, child_node, child_ctx, &child)); - } else { - RETURN_IF_ERROR(build_node_schema_with_mode(schema, child_node, child_ctx, &child, - SchemaBuildMode::NORMAL)); - } - child_types.push_back(make_nullable(child->type)); - child_names.push_back(child->name); - column_schema->children.push_back(std::move(child)); +void propagate_native_max_levels(ParquetColumnSchema* schema) { + DORIS_CHECK(schema != nullptr); + for (const auto& child : schema->children) { + DORIS_CHECK(child != nullptr); + propagate_native_max_levels(child.get()); + schema->max_definition_level = + std::max(schema->max_definition_level, child->max_definition_level); + schema->max_repetition_level = + std::max(schema->max_repetition_level, child->max_repetition_level); } - column_schema->type = - nullable_if_needed(std::make_shared(child_types, child_names), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); } -Status build_node_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, const SchemaBuildContext& context, - std::unique_ptr* result) { - return build_node_schema_with_mode(schema, node, context, result, SchemaBuildMode::NORMAL); +std::unique_ptr build_native_node_schema(const NativeFieldSchema& field, + int32_t local_id) { + auto result = std::make_unique(); + result->local_id = local_id; + result->parquet_field_id = field.field_id; + result->name = field.name; + result->type = field.data_type; + result->definition_level = field.definition_level; + result->repetition_level = field.repetition_level; + result->max_definition_level = field.definition_level; + result->max_repetition_level = field.repetition_level; + result->nullable_definition_level = field.data_type != nullptr && field.data_type->is_nullable() + ? field.definition_level - field.repetition_level + : 0; + result->repeated_ancestor_definition_level = field.repeated_parent_def_level; + result->repeated_repetition_level = field.repetition_level; + + const auto primitive_type = remove_nullable(field.data_type)->get_primitive_type(); + if (field.children.empty()) { + result->kind = ParquetColumnSchemaKind::PRIMITIVE; + result->leaf_column_id = field.physical_column_index; + fill_native_type_descriptor(field, &result->type_descriptor); + return result; + } + if (primitive_type == TYPE_ARRAY) { + result->kind = ParquetColumnSchemaKind::LIST; + } else if (primitive_type == TYPE_MAP) { + result->kind = ParquetColumnSchemaKind::MAP; + } else { + result->kind = ParquetColumnSchemaKind::STRUCT; + } + result->children.reserve(field.children.size()); + for (size_t child_idx = 0; child_idx < field.children.size(); ++child_idx) { + result->children.push_back( + build_native_node_schema(field.children[child_idx], cast_set(child_idx))); + } + propagate_native_max_levels(result.get()); + return result; } } // namespace -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields) { if (fields == nullptr) { return Status::InvalidArgument("fields is null"); } fields->clear(); - const auto* root = schema.group_node(); - if (root == nullptr) { - return Status::InvalidArgument("Parquet schema root is null"); - } - fields->reserve(root->field_count()); - for (int field_idx = 0; field_idx < root->field_count(); ++field_idx) { - std::unique_ptr field; - SchemaBuildContext context; - RETURN_IF_ERROR(build_node_schema( - schema, *root->field(field_idx), - child_context(context, *root->field(field_idx), field_idx), &field)); - fields->push_back(std::move(field)); + const auto& native_fields = schema.get_fields_schema(); + fields->reserve(native_fields.size()); + for (size_t field_idx = 0; field_idx < native_fields.size(); ++field_idx) { + // Unsupported logical leaves stay in the file schema so request-level validation can + // ignore unprojected fields and COUNT(*) placeholders without weakening real projections. + // The scan projection and native readers must share one tree; rebuilding wrappers through + // Arrow changes legacy LIST/STRUCT boundaries and makes valid nested values look absent. + fields->push_back( + build_native_node_schema(native_fields[field_idx], cast_set(field_idx))); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_column_schema.h b/be/src/format_v2/parquet/parquet_column_schema.h index 1fb7262aabde6f..697bcd498382b1 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.h +++ b/be/src/format_v2/parquet/parquet_column_schema.h @@ -23,18 +23,15 @@ #include "core/data_type/data_type.h" #include "format_v2/parquet/parquet_type.h" -namespace parquet { -class ColumnDescriptor; -class SchemaDescriptor; -} // namespace parquet - namespace doris::format::parquet { +class NativeFieldDescriptor; + enum class ParquetColumnSchemaKind { - PRIMITIVE, // primitive leaf -> ScalarColumnReader - STRUCT, // struct -> StructColumnReader - LIST, // array -> ListColumnReader - MAP, // map -> MapColumnReader + PRIMITIVE, // physical primitive leaf + STRUCT, // Parquet group with STRUCT semantics + LIST, // Parquet group with LIST semantics + MAP, // Parquet group with MAP semantics }; // ============================================================================ @@ -55,8 +52,6 @@ struct ParquetColumnSchema { ParquetColumnSchemaKind kind = ParquetColumnSchemaKind::PRIMITIVE; - const ::parquet::ColumnDescriptor* descriptor = nullptr; - // ======== Dremel Levels ======== int16_t max_definition_level = 0; @@ -74,7 +69,7 @@ struct ParquetColumnSchema { std::vector> children {}; }; -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields); } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 52151c7942af2b..c178e491b8ac21 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -15,86 +15,139 @@ #include "format_v2/parquet/parquet_file_context.h" -#include -#include #include -#include -#include #include #include #include #include -#include #include -#include #include +#include "common/cast_set.h" #include "common/check.h" #include "common/config.h" +#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" -#include "storage/cache/page_cache.h" +#include "runtime/exec_env.h" +#include "util/coding.h" #include "util/slice.h" +#include "util/thrift_util.h" +#include "util/time.h" namespace doris::format::parquet { +constexpr size_t V2_PARQUET_FOOTER_SIZE = 8; + +NativeParquetMetadata::NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size) + : _metadata(std::move(metadata)), _parsed_size(parsed_size) { + ExecEnv::GetInstance()->parquet_meta_tracker()->consume(get_mem_size()); +} + +NativeParquetMetadata::~NativeParquetMetadata() { + ExecEnv::GetInstance()->parquet_meta_tracker()->release(get_mem_size()); +} + +Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + _schema.set_enable_mapping_varbinary(enable_mapping_varbinary); + _schema.set_enable_mapping_timestamp_tz(enable_mapping_timestamp_tz); + RETURN_IF_ERROR(_schema.parse_from_thrift(_metadata.schema)); + // Native readers address projected leaves by stable DFS IDs. Assign them only on the private + // v2 schema object so v1's cached schema lifecycle and numbering remain untouched. + _schema.assign_ids(); + for (size_t row_group_idx = 0; row_group_idx < _metadata.row_groups.size(); ++row_group_idx) { + const auto& row_group = _metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.columns.size() != _schema.physical_fields_size()) { + // All v2 planners index chunks by the native DFS leaf order, so validate cardinality + // once before any projection, prefetch, or decoder can perform indexed access. + return Status::Corruption( + "Parquet row group {} has {} column chunks but schema has {} physical fields", + row_group_idx, row_group.columns.size(), _schema.physical_fields_size()); + } + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + const auto* physical_field = _schema.get_physical_field(column_idx); + if (chunk.meta_data.type != physical_field->physical_type) { + return Status::Corruption( + "Parquet row group {} column {} physical type {} does not match schema {}", + row_group_idx, column_idx, tparquet::to_string(chunk.meta_data.type), + tparquet::to_string(physical_field->physical_type)); + } + if (chunk.meta_data.num_values < 0) { + return Status::Corruption( + "Parquet row group {} column {} has negative value count {}", row_group_idx, + column_idx, chunk.meta_data.num_values); + } + if (physical_field->repetition_level == 0 && + chunk.meta_data.num_values != row_group.num_rows) { + // A flat leaf contributes one definition-level slot per row, including NULLs. + // Reject a mismatched footer before metadata COUNT can treat it as cardinality. + return Status::Corruption( + "Parquet row group {} flat column {} has {} values but {} rows", + row_group_idx, column_idx, chunk.meta_data.num_values, row_group.num_rows); + } + } + } + return Status::OK(); +} + namespace detail { -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges) { - if (position < 0 || nbytes <= 0) { - return {}; +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit) { + if (file_size < V2_PARQUET_FOOTER_SIZE || + serialized_size > file_size - V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 footer size {} exceeds file size {}", serialized_size, + file_size); + } + if (serialized_size > metadata_size_limit) { + return Status::Corruption("Parquet v2 footer size {} exceeds metadata limit {}", + serialized_size, metadata_size_limit); } + return Status::OK(); +} - std::vector ranges; - ranges.reserve(cached_ranges.size()); - const int64_t request_end = position + nbytes; - for (const auto& range : cached_ranges) { - if (range.size > 0 && range.offset < request_end && position < range.end_offset()) { - ranges.push_back(range); - } +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable) { + const int64_t mtime = description_mtime != 0 ? description_mtime : reader_mtime; + if (mtime == 0 && !is_immutable) { + // Unknown version is not a stable identity: an overwrite can preserve path and size while + // changing both footer semantics and page bytes. + return {}; } - std::sort(ranges.begin(), ranges.end(), [](const auto& lhs, const auto& rhs) { - if (lhs.offset != rhs.offset) { - return lhs.offset < rhs.offset; - } - return lhs.size > rhs.size; - }); - - std::vector plan; - int64_t cursor = position; - while (cursor < request_end) { - // At each cursor position, choose the cached range that already covers the cursor and - // extends farthest to the right. This handles both adjacent ranges and overlapping - // ranges. If no range covers the current cursor, there is a gap and the request must - // miss as a whole. - auto best = ranges.end(); - int64_t best_end = cursor; - for (auto it = ranges.begin(); it != ranges.end(); ++it) { - const int64_t cached_end = it->end_offset(); - if (it->offset <= cursor && cursor < cached_end && cached_end > best_end) { - best = it; - best_end = cached_end; - } - } - if (best == ranges.end()) { - return {}; - } - const int64_t copy_size = std::min(best_end, request_end) - cursor; - ParquetPageCacheReadPlanEntry entry; - entry.cached_range = *best; - entry.copy_offset_in_cache = cursor - best->offset; - entry.output_offset = cursor - position; - entry.copy_size = copy_size; - plan.push_back(entry); - cursor += copy_size; + const int64_t file_size = description_file_size >= 0 ? description_file_size : reader_file_size; + return fmt::format("{}::{}::mtime={}::size={}", fs_name, path, mtime, file_size); +} + +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length) { + if (offset < 0 || length <= 0 || length > MAX_SERIALIZED_PARQUET_INDEX_BYTES || + static_cast(offset) > file_size) { + return false; } - return plan; + return static_cast(length) <= file_size - static_cast(offset); +} + +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end) { + return span_offset >= 0 && span_end >= span_offset && + span_end - span_offset <= MAX_SERIALIZED_PARQUET_INDEX_BYTES; } std::vector valid_prefetch_ranges( @@ -129,489 +182,426 @@ bool should_use_merge_range_reader(const std::vector& ran avg_io_size < io::MergeRangeFileReader::SMALL_IO; } +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size) { + return file_size <= in_memory_file_size && + (path.starts_with("http://") || path.starts_with("https://")); +} + } // namespace detail namespace { -// StoragePageCache only supports exact-key lookup. Keep lightweight range metadata here so later -// Arrow ReadAt requests can reuse cached bytes when their requested ranges are subsets of, or are -// fully covered by, previously cached ranges. Stale metadata is pruned on lookup. -std::mutex cached_page_range_index_mutex; -std::unordered_map> cached_page_range_index; -constexpr size_t MAX_CACHED_PAGE_RANGE_FILES = 4096; -constexpr size_t MAX_CACHED_PAGE_RANGES_PER_FILE = 65536; - -void register_cached_page_range(const std::string& file_key, int64_t position, int64_t nbytes) { - DORIS_CHECK(nbytes > 0); - std::lock_guard lock(cached_page_range_index_mutex); - if (cached_page_range_index.find(file_key) == cached_page_range_index.end() && - cached_page_range_index.size() >= MAX_CACHED_PAGE_RANGE_FILES) { - cached_page_range_index.erase(cached_page_range_index.begin()); - } - auto& ranges = cached_page_range_index[file_key]; - auto it = std::find_if(ranges.begin(), ranges.end(), [&](const ParquetPageCacheRange& range) { - return range.offset == position && range.size == nbytes; - }); - if (it == ranges.end()) { - if (ranges.size() >= MAX_CACHED_PAGE_RANGES_PER_FILE) { - ranges.erase(ranges.begin()); - } - ranges.push_back(ParquetPageCacheRange {position, nbytes}); - } -} - -void unregister_cached_page_range(const std::string& file_key, - const ParquetPageCacheRange& stale_range) { - std::lock_guard lock(cached_page_range_index_mutex); - auto it = cached_page_range_index.find(file_key); - if (it == cached_page_range_index.end()) { - return; - } - auto& ranges = it->second; - ranges.erase(std::remove_if(ranges.begin(), ranges.end(), - [&](const ParquetPageCacheRange& range) { - return range.offset == stale_range.offset && - range.size == stale_range.size; - }), - ranges.end()); - if (ranges.empty()) { - cached_page_range_index.erase(it); - } -} - -std::vector cached_page_ranges_for_file(const std::string& file_key) { - std::lock_guard lock(cached_page_range_index_mutex); - auto it = cached_page_range_index.find(file_key); - if (it == cached_page_range_index.end()) { - return {}; - } - return it->second; +constexpr uint8_t V2_PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'}; +constexpr size_t V2_INITIAL_FOOTER_READ_SIZE = 48 * 1024; + +Status parse_native_parquet_footer(io::FileReaderSPtr file, + std::unique_ptr* metadata, + size_t* footer_size, io::IOContext* io_ctx, + bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(footer_size != nullptr); + const size_t file_size = file->size(); + if (file_size < V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 file is too small for a footer: {}", file_size); + } + + const size_t tail_size = std::min(file_size, V2_INITIAL_FOOTER_READ_SIZE); + std::vector tail(tail_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - tail_size, Slice(tail.data(), tail.size()), + &bytes_read, io_ctx)); + if (bytes_read != tail.size()) { + return Status::Corruption("Short Parquet v2 footer read: expected {}, got {}", tail.size(), + bytes_read); + } + const auto* magic = tail.data() + tail.size() - sizeof(V2_PARQUET_MAGIC); + if (memcmp(magic, V2_PARQUET_MAGIC, sizeof(V2_PARQUET_MAGIC)) != 0) { + return Status::Corruption("Invalid Parquet v2 footer magic in {}", file->path().native()); + } + + const uint32_t serialized_size = + decode_fixed32_le(tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE); + // The configured Thrift message ceiling also bounds this file-controlled allocation. Keep the + // check before both allocation and the optional second read so a sparse file cannot force a + // process-sized metadata buffer merely by advertising a large footer. + const size_t metadata_size_limit = + static_cast(std::max(config::thrift_max_message_size, 0)); + RETURN_IF_ERROR( + detail::validate_native_footer_size(serialized_size, file_size, metadata_size_limit)); + std::vector serialized_metadata(serialized_size); + if (serialized_size <= tail.size() - V2_PARQUET_FOOTER_SIZE) { + const auto* metadata_start = + tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE - serialized_size; + memcpy(serialized_metadata.data(), metadata_start, serialized_size); + } else { + bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - V2_PARQUET_FOOTER_SIZE - serialized_size, + Slice(serialized_metadata.data(), serialized_metadata.size()), + &bytes_read, io_ctx)); + if (bytes_read != serialized_metadata.size()) { + return Status::Corruption("Short Parquet v2 metadata read: expected {}, got {}", + serialized_metadata.size(), bytes_read); + } + } + + uint32_t thrift_size = serialized_size; + tparquet::FileMetaData thrift_metadata; + RETURN_IF_ERROR(deserialize_thrift_msg(serialized_metadata.data(), &thrift_size, true, + &thrift_metadata)); + auto parsed = + std::make_unique(std::move(thrift_metadata), serialized_size); + RETURN_IF_ERROR(parsed->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz)); + *footer_size = V2_PARQUET_FOOTER_SIZE + serialized_size; + *metadata = std::move(parsed); + return Status::OK(); } std::string build_page_cache_file_key(const io::FileReader& file_reader, const io::FileDescription& file_description) { - const int64_t mtime = - file_description.mtime != 0 ? file_description.mtime : file_reader.mtime(); - if (mtime == 0 && !file_description.is_immutable) { - // mtime == 0 means "unknown version", not the Unix epoch. V1 historically caches such a - // file under path::0, but copying that behavior for every V2 file is unsafe: a mutable file - // can be overwritten with different bytes while retaining both its path and size, causing - // process-global page cache entries to return stale data. Only callers that explicitly - // guarantee path immutability may use the mtime=0 cache key below. - return {}; - } - const int64_t file_size = file_description.file_size >= 0 - ? file_description.file_size - : static_cast(file_reader.size()); - return fmt::format("{}::{}::mtime={}::size={}", file_description.fs_name, - file_reader.path().native(), mtime, file_size); + return detail::build_native_file_cache_key( + file_description.fs_name, file_reader.path().native(), file_description.mtime, + file_reader.mtime(), file_description.file_size, + static_cast(file_reader.size()), file_description.is_immutable); } -class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { -public: - DorisRandomAccessFile(io::FileReaderSPtr file_reader, io::IOContext* io_ctx, - bool enable_page_cache, std::string page_cache_file_key) - : _file_reader(std::move(file_reader)), - _base_file_reader(_file_reader), - _io_ctx(io_ctx), - _enable_page_cache(enable_page_cache), - _page_cache_file_key(std::move(page_cache_file_key)) { - DORIS_CHECK(_file_reader != nullptr); - if (auto tracing_reader = std::dynamic_pointer_cast(_file_reader)) { - _file_reader_stats = tracing_reader->stats(); - _base_file_reader = tracing_reader->inner_reader(); - } - DORIS_CHECK(_base_file_reader != nullptr); - set_mode(arrow::io::FileMode::READ); - } - - arrow::Status Close() override { - if (!_closed) { - collect_active_merge_range_profile(); - _closed = true; - } - return arrow::Status::OK(); - } - - bool closed() const override { return _closed; } - - arrow::Result Tell() const override { return _pos; } - - arrow::Status Seek(int64_t position) override { - if (position < 0) { - return arrow::Status::Invalid("negative seek position"); - } - _pos = position; - return arrow::Status::OK(); - } - - arrow::Result GetSize() override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - return static_cast(_file_reader->size()); - } - - arrow::Result Read(int64_t nbytes, void* out) override { - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(_pos, nbytes, out)); - _pos += bytes_read; - return bytes_read; - } - - arrow::Result> Read(int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, Read(nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; - } +} // namespace - arrow::Result ReadAt(int64_t position, int64_t nbytes, void* out) override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - if (position < 0 || nbytes < 0) { - return arrow::Status::Invalid("negative read position or length"); - } - if (try_read_from_page_cache(position, nbytes, out)) { - return nbytes; - } - size_t bytes_read = 0; - Status st = _file_reader->read_at( - static_cast(position), - Slice(static_cast(out), static_cast(nbytes)), &bytes_read, - _io_ctx); - if (!st.ok()) { - return arrow::Status::IOError(st.to_string_no_stack()); - } - insert_page_cache(position, nbytes, out, bytes_read); - return static_cast(bytes_read); - } +Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, + bool enable_page_cache, const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) { + DORIS_CHECK(input_file_reader != nullptr); + if (detail::should_stage_small_http_file(input_file_reader->path().native(), + input_file_reader->size(), + config::in_memory_file_size)) { + // A metadata-cache hit can make the first physical read start inside a tiny HTTP file. + // Read it from byte zero once so EOF-range quirks cannot make warm scans less reliable + // than cold scans, while keeping this compatibility policy entirely inside v2. + native_file = std::make_shared(std::move(input_file_reader)); + } else { + native_file = std::move(input_file_reader); + } + native_io_ctx = io_ctx; + + // Footer and page bytes must use the same stable identity. In particular, fs_name separates + // identical HDFS paths from different nameservices, while an unknown mutable version bypasses + // both caches rather than reusing an overwritten file of the same size. + auto* meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + auto meta_cache_key = build_page_cache_file_key(*native_file, file_description); + const bool has_stable_meta_cache_identity = !meta_cache_key.empty(); + if (has_stable_meta_cache_identity) { + // The discriminator prevents a v1 metadata value from being cast as the v2-owned type; + // schema mapping flags participate because they change the cached native schema tree. + meta_cache_key.append("\0v2", 3); + meta_cache_key.push_back(static_cast(enable_mapping_varbinary)); + meta_cache_key.push_back(static_cast(enable_mapping_timestamp_tz)); + } + size_t native_footer_size = 0; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled() && + meta_cache->lookup(meta_cache_key, &native_meta_cache_handle)) { + native_metadata = native_meta_cache_handle.data(); + ++native_footer_cache_hits; + } else { + RETURN_IF_ERROR(parse_native_parquet_footer( + native_file, &native_metadata_owner, &native_footer_size, io_ctx, + enable_mapping_varbinary, enable_mapping_timestamp_tz)); + ++native_footer_read_calls; + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled()) { + meta_cache->insert(meta_cache_key, native_metadata_owner.release(), + &native_meta_cache_handle); + native_metadata = native_meta_cache_handle.data(); + } else { + native_metadata = native_metadata_owner.get(); + } + } + DORIS_CHECK(native_metadata != nullptr); + + auto page_cache_file_key = build_page_cache_file_key(*native_file, file_description); + native_page_cache_enabled = enable_page_cache && !page_cache_file_key.empty(); + // Native page readers use the FileDescription-derived immutable identity directly. + native_page_cache_file_key = page_cache_file_key; + return Status::OK(); +} - arrow::Result> ReadAt(int64_t position, - int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(position, nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; +Status ParquetFileContext::load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const { + DORIS_CHECK(offset_indexes != nullptr); + offset_indexes->clear(); + if (leaf_column_ids.empty()) { + return Status::OK(); } - - void register_page_cache_ranges(std::vector ranges) { - std::lock_guard lock(_page_cache_mutex); - _page_cache_ranges = std::move(ranges); + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for OffsetIndex", row_group_id); } - - void prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { - auto cached_reader = cached_remote_file_reader(); - if (cached_reader == nullptr) { - return; - } - const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : _io_ctx; - for (const auto& range : ranges) { - if (range.offset < 0 || range.size <= 0) { + const auto& native_row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + try { + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(native_row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for OffsetIndex", + leaf_column_id); + } + const auto& column_chunk = native_row_group.columns[leaf_column_id]; + if (!column_chunk.__isset.offset_index_offset || + !column_chunk.__isset.offset_index_length || + column_chunk.offset_index_length <= 0) { continue; } - cached_reader->prefetch_range(static_cast(range.offset), - static_cast(range.size), prefetch_io_ctx); - } - } - - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - reset_active_file_reader(); - const auto valid_ranges = detail::valid_prefetch_ranges(ranges); - if (!detail::should_use_merge_range_reader( - valid_ranges, avg_io_size, - typeid_cast(_base_file_reader.get()) != nullptr)) { - return false; - } - - std::vector random_access_ranges; - random_access_ranges.reserve(valid_ranges.size()); - for (const auto& range : valid_ranges) { - random_access_ranges.emplace_back(static_cast(range.offset), - static_cast(range.end_offset())); - } - - // This mirrors the v1 parquet reader: when projected column chunks in a row group are - // small random IOs, make the actual ReadAt path range-aware. Arrow still drives decoding, - // but every page read below this point sees MergeRangeFileReader instead of the raw remote - // reader, so adjacent small requests can be coalesced and served from merge buffers. - // Example: a row group projects leaf chunks [1MB, 1.5MB) and [1.6MB, 2MB). Arrow later - // issues page reads inside those chunks; MergeRangeFileReader can fetch a wider slice once - // and satisfy the following ReadAt calls from its boxes, reducing remote request count. - _merge_range_active = true; - set_active_file_reader(std::make_shared( - profile, _base_file_reader, random_access_ranges, merge_read_slice_size)); - return true; - } - - void reset_random_access_ranges() { reset_active_file_reader(); } - - ParquetPageCacheStats page_cache_stats() const { - std::lock_guard lock(_page_cache_mutex); - return _page_cache_stats; - } - -private: - bool page_cache_enabled() const { - return _enable_page_cache && !config::disable_storage_page_cache && - StoragePageCache::instance() != nullptr && !_page_cache_file_key.empty(); - } - - bool range_in_page_cache_scope(int64_t position, int64_t nbytes) const { - if (nbytes <= 0) { - return false; - } - const int64_t end = position + nbytes; - for (const auto& range : _page_cache_ranges) { - const int64_t range_end = range.offset + range.size; - if (position >= range.offset && end <= range_end) { - return true; + const int64_t index_offset = column_chunk.offset_index_offset; + const int64_t index_length = column_chunk.offset_index_length; + if (!detail::is_serialized_index_range_safe(native_file->size(), index_offset, + index_length)) { + // OffsetIndex is optional. A malformed range must not allocate from untrusted + // footer values or redirect the native reader outside the file. + continue; + } + std::vector serialized_index(static_cast(index_length)); + Slice index_slice(serialized_index.data(), serialized_index.size()); + size_t bytes_read = 0; + if (!native_file->read_at(index_offset, index_slice, &bytes_read, native_io_ctx).ok() || + bytes_read != serialized_index.size()) { + continue; + } + uint32_t thrift_length = static_cast(serialized_index.size()); + tparquet::OffsetIndex native_index; + if (!deserialize_thrift_msg(serialized_index.data(), &thrift_length, true, + &native_index) + .ok() || + native_index.page_locations.empty()) { + continue; + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + native_row_group.columns[leaf_column_id].meta_data, native_file->size(), + compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index( + native_index, chunk_range, + native_row_group.columns[leaf_column_id].meta_data.data_page_offset, + native_row_group.num_rows)) { + // OffsetIndex is optional. Reject the complete index instead of letting one bad + // location redirect an indexed reader outside its owning column chunk. + continue; } + offset_indexes->emplace(leaf_column_id, std::move(native_index)); } - return false; + } catch (const std::exception&) { + // OffsetIndex is optional. Selected logical ranges still enforce correctness, while the + // native reader conservatively falls back to sequential page traversal. + offset_indexes->clear(); } + return Status::OK(); +} - StoragePageCache::CacheKey page_cache_key(int64_t position, int64_t nbytes) const { - return StoragePageCache::CacheKey(_page_cache_file_key, - static_cast(position + nbytes), position); +Status ParquetFileContext::load_native_page_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, int64_t* read_time, + int64_t* parse_time) const { + DORIS_CHECK(page_indexes != nullptr); + page_indexes->clear(); + if (leaf_column_ids.empty()) { + return Status::OK(); } - - bool copy_cached_range(const ParquetPageCacheRange& cached_range, int64_t copy_position, - int64_t copy_size, void* out, int64_t output_offset) { - PageCacheHandle handle; - if (!StoragePageCache::instance()->lookup( - page_cache_key(cached_range.offset, cached_range.size), &handle, - segment_v2::DATA_PAGE)) { - unregister_cached_page_range(_page_cache_file_key, cached_range); - return false; + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for PageIndex", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + + struct SerializedIndexRange { + int leaf_column_id; + int64_t offset; + int64_t length; + }; + struct PendingPageIndex { + NativeParquetPageIndex indexes; + bool has_column_index = false; + bool has_offset_index = false; + }; + std::vector column_index_ranges; + std::vector offset_index_ranges; + std::unordered_map pending_indexes; + + auto valid_index_range = [&](int64_t offset, int64_t length) { + return detail::is_serialized_index_range_safe(native_file->size(), offset, length); + }; + + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for PageIndex", leaf_column_id); + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (!chunk.__isset.column_index_offset || !chunk.__isset.column_index_length || + !chunk.__isset.offset_index_offset || !chunk.__isset.offset_index_length) { + continue; } - Slice cached = handle.data(); - const int64_t cache_offset = copy_position - cached_range.offset; - DORIS_CHECK(cache_offset >= 0); - DORIS_CHECK(cached.size >= static_cast(cache_offset + copy_size)); - memcpy(static_cast(out) + output_offset, cached.data + cache_offset, - static_cast(copy_size)); - return true; - } - - bool try_read_from_cached_ranges(int64_t position, int64_t nbytes, void* out) { - auto plan = detail::plan_page_cache_range_read( - position, nbytes, cached_page_ranges_for_file(_page_cache_file_key)); - if (plan.empty()) { - return false; + if (!valid_index_range(chunk.column_index_offset, chunk.column_index_length) || + !valid_index_range(chunk.offset_index_offset, chunk.offset_index_length)) { + continue; } - for (const auto& entry : plan) { - if (!copy_cached_range(entry.cached_range, - entry.cached_range.offset + entry.copy_offset_in_cache, - entry.copy_size, out, entry.output_offset)) { - return false; + column_index_ranges.push_back( + {leaf_column_id, chunk.column_index_offset, chunk.column_index_length}); + offset_index_ranges.push_back( + {leaf_column_id, chunk.offset_index_offset, chunk.offset_index_length}); + pending_indexes.try_emplace(leaf_column_id); + } + + auto read_coalesced_indexes = [&](std::vector* ranges, + bool column_index) { + std::sort(ranges->begin(), ranges->end(), + [](const auto& lhs, const auto& rhs) { return lhs.offset < rhs.offset; }); + size_t range_begin = 0; + while (range_begin < ranges->size()) { + size_t range_end = range_begin + 1; + int64_t span_end = (*ranges)[range_begin].offset + (*ranges)[range_begin].length; + while (range_end < ranges->size() && (*ranges)[range_end].offset <= span_end) { + span_end = std::max(span_end, + (*ranges)[range_end].offset + (*ranges)[range_end].length); + ++range_end; } - } - return true; - } - bool try_read_from_page_cache(int64_t position, int64_t nbytes, void* out) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes)) { - return false; - } - ++_page_cache_stats.read_count; - // Fast path: Arrow issues the same ReadAt(offset, size) again, so the exact - // StoragePageCache key matches. - // Fallback path: Arrow may read a different but related byte range on another scan. - // Examples: - // - Current request [120, 150) can be served from cached [100, 200) by copying the - // 30-byte subset starting at cached offset 20. - // - Current request [100, 260) can be served by stitching cached [100, 180) and - // [180, 260). If any middle span is missing, it is a miss and the file reader fills - // the whole request from storage. - if (!copy_cached_range(ParquetPageCacheRange {position, nbytes}, position, nbytes, out, - 0) && - !try_read_from_cached_ranges(position, nbytes, out)) { - ++_page_cache_stats.miss_count; - return false; + const int64_t span_offset = (*ranges)[range_begin].offset; + if (!detail::is_serialized_index_span_safe(span_offset, span_end)) { + // Optional indexes share one allocation per contiguous footer block. Skipping the + // whole block prevents many individually small ranges from bypassing the budget. + range_begin = range_end; + continue; + } + const int64_t span_length = span_end - span_offset; + std::vector serialized(static_cast(span_length)); + Slice slice(serialized.data(), serialized.size()); + size_t bytes_read = 0; + Status read_status; + int64_t read_time_sink = 0; + { + SCOPED_RAW_TIMER(read_time == nullptr ? &read_time_sink : read_time); + read_status = native_file->read_at(span_offset, slice, &bytes_read, native_io_ctx); + } + if (read_status.ok() && bytes_read == serialized.size()) { + for (size_t i = range_begin; i < range_end; ++i) { + const auto& range = (*ranges)[i]; + auto pending = pending_indexes.find(range.leaf_column_id); + if (pending == pending_indexes.end()) { + continue; + } + uint32_t thrift_length = static_cast(range.length); + const auto* thrift_data = + serialized.data() + static_cast(range.offset - span_offset); + int64_t parse_time_sink = 0; + SCOPED_RAW_TIMER(parse_time == nullptr ? &parse_time_sink : parse_time); + if (column_index) { + pending->second.has_column_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.column_index) + .ok(); + } else { + pending->second.has_offset_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.offset_index) + .ok(); + } + } + } + range_begin = range_end; } - ++_page_cache_stats.hit_count; - ++_page_cache_stats.compressed_hit_count; - return true; - } + }; - void insert_page_cache(int64_t position, int64_t nbytes, const void* data, size_t bytes_read) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes) || - bytes_read != static_cast(nbytes)) { - return; - } - auto* page = new DataPage(bytes_read, true, segment_v2::DATA_PAGE); - memcpy(page->data(), data, bytes_read); - PageCacheHandle handle; - StoragePageCache::instance()->insert(page_cache_key(position, nbytes), page, &handle, - segment_v2::DATA_PAGE); - register_cached_page_range(_page_cache_file_key, position, nbytes); - ++_page_cache_stats.write_count; - ++_page_cache_stats.compressed_write_count; - } + // Parquet writers place each index kind in a contiguous block. Reading overlapping/adjacent + // ranges as one span keeps cold small-file planning from paying two remote round trips per + // projected leaf, while refusing gaps avoids amplifying reads from untrusted footer offsets. + read_coalesced_indexes(&column_index_ranges, true); + read_coalesced_indexes(&offset_index_ranges, false); - void set_active_file_reader(io::FileReaderSPtr reader) { - DORIS_CHECK(reader != nullptr); - _file_reader = _file_reader_stats != nullptr - ? std::make_shared(std::move(reader), - _file_reader_stats) - : std::move(reader); - } - - void reset_active_file_reader() { - collect_active_merge_range_profile(); - _merge_range_active = false; - set_active_file_reader(_base_file_reader); - } - - void collect_active_merge_range_profile() { - if (_merge_range_active && _file_reader != nullptr) { - // MergeRangeFileReader writes its MergedSmallIO counters only from - // collect_profile_before_close(). v2 replaces the active reader for every row group, - // so collect before overwriting it; Close() handles the final row group. Example: - // RG0 installs a merge reader, RG1 calls set_random_access_ranges() and resets the - // active reader first, so RG0's RequestIO/MergedIO counters must be flushed here. - _file_reader->collect_profile_before_close(); - } - } - - std::shared_ptr cached_remote_file_reader() { - if (_merge_range_active) { - return nullptr; - } - auto reader = _file_reader; - if (reader == nullptr) { - return nullptr; + for (auto& [leaf_column_id, pending] : pending_indexes) { + const auto& chunk = row_group.columns[leaf_column_id]; + auto& indexes = pending.indexes; + if (!pending.has_column_index || !pending.has_offset_index || + indexes.column_index.null_pages.size() != indexes.offset_index.page_locations.size()) { + continue; } - // FileReader::init wraps the physical reader with TracingFileReader when scan IO stats are - // enabled. Prefetch should target the physical cached reader below that tracing wrapper, - // otherwise v2 scans with profiling would silently lose prefetch. - if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { - reader = tracing_reader->inner_reader(); + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, native_file->size(), compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index(indexes.offset_index, chunk_range, + chunk.meta_data.data_page_offset, row_group.num_rows)) { + continue; } - return std::dynamic_pointer_cast(reader); + page_indexes->emplace(leaf_column_id, std::move(indexes)); } + return Status::OK(); +} - io::FileReaderSPtr _file_reader; - io::FileReaderSPtr _base_file_reader; - io::FileReaderStats* _file_reader_stats = nullptr; - io::IOContext* _io_ctx = nullptr; - int64_t _pos = 0; - bool _closed = false; - bool _enable_page_cache = false; - bool _merge_range_active = false; - std::string _page_cache_file_key; - mutable std::mutex _page_cache_mutex; - std::vector _page_cache_ranges; - ParquetPageCacheStats _page_cache_stats; -}; - -} // namespace - -Status arrow_status_to_doris_status(const arrow::Status& status) { - if (status.ok()) { - return Status::OK(); +void ParquetFileContext::prefetch_ranges(const std::vector& ranges, + const io::IOContext* io_ctx) { + io::FileReaderSPtr reader = native_file; + if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { + reader = tracing_reader->inner_reader(); } - if (status.IsIOError()) { - return Status::IOError(status.ToString()); + auto cached_reader = std::dynamic_pointer_cast(reader); + if (cached_reader == nullptr) { + return; } - if (status.IsInvalid()) { - return Status::InvalidArgument(status.ToString()); + const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : native_io_ctx; + for (const auto& range : detail::valid_prefetch_ranges(ranges)) { + cached_reader->prefetch_range(cast_set(range.offset), cast_set(range.size), + prefetch_io_ctx); } - return Status::InternalError(status.ToString()); } -Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, - bool enable_page_cache, - const io::FileDescription& file_description) { - DORIS_CHECK(input_file_reader != nullptr); - auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); - arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, - enable_page_cache, - std::move(page_cache_file_key)); - try { - // TODO: Cache parquet metadata in file system layer to avoid repeated metadata read for same file. - this->file_reader = ::parquet::ParquetFileReader::Open( - arrow_file, ::parquet::default_reader_properties()); - metadata = this->file_reader->metadata(); - schema = metadata != nullptr ? metadata->schema() : nullptr; - } catch (const ::parquet::ParquetException& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet file: {}", e.what()); - } catch (const std::exception& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet file: {}", e.what()); +bool ParquetFileContext::set_native_random_access_ranges( + const std::vector& ranges, size_t avg_io_size, + RuntimeProfile* profile, int64_t merge_read_slice_size) { + DORIS_CHECK(native_file != nullptr); + if (!detail::should_use_merge_range_reader( + ranges, avg_io_size, + typeid_cast(native_file.get()) != nullptr)) { + native_row_group_file = native_file; + return false; } - if (metadata == nullptr || schema == nullptr) { - return Status::Corruption("Failed to read parquet metadata"); + const auto valid_ranges = detail::valid_prefetch_ranges(ranges); + std::vector native_ranges; + native_ranges.reserve(valid_ranges.size()); + for (const auto& range : valid_ranges) { + native_ranges.emplace_back(cast_set(range.offset), + cast_set(range.end_offset())); } - return Status::OK(); -} - -void ParquetFileContext::register_page_cache_ranges(std::vector ranges) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get()) - ->register_page_cache_ranges(std::move(ranges)); -} - -void ParquetFileContext::prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->prefetch_ranges(ranges, io_ctx); -} - -bool ParquetFileContext::set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - DORIS_CHECK(arrow_file != nullptr); - return static_cast(arrow_file.get()) - ->set_random_access_ranges(ranges, avg_io_size, profile, merge_read_slice_size); + std::ranges::sort(native_ranges, {}, &io::PrefetchRange::start_offset); + native_row_group_file = std::make_shared( + profile, native_file, native_ranges, merge_read_slice_size); + return true; } void ParquetFileContext::reset_random_access_ranges() { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->reset_random_access_ranges(); + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); } ParquetPageCacheStats ParquetFileContext::page_cache_stats() const { - if (arrow_file == nullptr) { - return {}; - } - return static_cast(arrow_file.get())->page_cache_stats(); + return {}; } Status ParquetFileContext::close() { - if (file_reader != nullptr) { - try { - file_reader->Close(); - } catch (const std::exception&) { - } - } - if (arrow_file != nullptr) { - static_cast(arrow_status_to_doris_status(arrow_file->Close())); - } - file_reader.reset(); - arrow_file.reset(); + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); + native_metadata = nullptr; + native_metadata_owner.reset(); + native_meta_cache_handle = {}; + native_file.reset(); + native_io_ctx = nullptr; + native_page_cache_enabled = false; + native_page_cache_file_key.clear(); return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 0dca52244957d7..0cd413e10557ac 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -15,16 +15,21 @@ #pragma once -#include -#include +#include #include #include #include +#include +#include +#include +#include #include #include "common/status.h" +#include "format_v2/parquet/native_schema_desc.h" #include "io/fs/file_reader.h" +#include "util/obj_lru_cache.h" namespace doris::io { struct FileDescription; @@ -37,6 +42,26 @@ class RuntimeProfile; namespace doris::format::parquet { +struct NativeParquetPageIndex; + +// V2-owned footer/schema tree. Production planning and decoding consume this object directly; +// Arrow metadata is intentionally not materialized from the serialized footer. +class NativeParquetMetadata { +public: + NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size); + ~NativeParquetMetadata(); + + Status init_schema(bool enable_mapping_varbinary, bool enable_mapping_timestamp_tz); + const tparquet::FileMetaData& to_thrift() const { return _metadata; } + const NativeFieldDescriptor& schema() const { return _schema; } + size_t get_mem_size() const { return _parsed_size; } + +private: + tparquet::FileMetaData _metadata; + NativeFieldDescriptor _schema; + size_t _parsed_size = 0; +}; + struct ParquetPageCacheRange { int64_t offset = 0; int64_t size = 0; @@ -44,17 +69,6 @@ struct ParquetPageCacheRange { int64_t end_offset() const { return offset + size; } }; -struct ParquetPageCacheReadPlanEntry { - // The exact cached StoragePageCache entry. The final cache key is still exact-range based: - // file key + cached_range.end_offset() + cached_range.offset. - ParquetPageCacheRange cached_range; - // Byte offset inside cached_range to start copying from. - int64_t copy_offset_in_cache = 0; - // Byte offset inside the current ReadAt output buffer to start writing to. - int64_t output_offset = 0; - int64_t copy_size = 0; -}; - struct ParquetPageCacheStats { int64_t read_count = 0; int64_t write_count = 0; @@ -66,21 +80,19 @@ struct ParquetPageCacheStats { namespace detail { -// Build the copy plan for a ReadAt(position, nbytes) request from the range metadata of -// previously cached entries. -// StoragePageCache cannot do range lookup by itself; it can only lookup an exact key. The -// caller therefore keeps lightweight cached range metadata and uses this function to decide -// which exact cache entries to fetch and which byte spans to copy. -// Examples: -// 1. Subset hit: -// request [120, 150), cached [100, 200) -> copy 30 bytes from cached offset 20. -// 2. Superset hit covered by multiple cached entries: -// request [100, 260), cached [100, 180) and [180, 260) -// -> two copies: [100, 180) to output offset 0, [180, 260) to output offset 80. -// 3. Partial overlap is a miss: -// request [100, 260), cached [100, 180) only -> empty plan, caller reads from file. -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges); +inline constexpr int64_t MAX_SERIALIZED_PARQUET_INDEX_BYTES = 64LL << 20; + +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit); + +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable); + +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length); + +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end); // Keep only byte ranges that are safe to hand to FileReader implementations. Parquet metadata is // expected to contain non-negative offsets and positive compressed sizes, but tests and corrupted @@ -89,54 +101,74 @@ std::vector plan_page_cache_range_read( std::vector valid_prefetch_ranges( const std::vector& ranges); -// Average projected column-chunk size for one row group. The v1 parquet path uses this value to -// decide whether a row group is dominated by small random IOs; v2 uses the same signal before -// installing MergeRangeFileReader. Example: chunks of 512KB and 1MB average below SMALL_IO and are -// good merge-reader candidates, while two 8MB chunks should stay on the raw random-access reader. +// Average projected column-chunk size for one row group. V2 uses this signal to decide whether the +// row group is dominated by small random IOs before installing MergeRangeFileReader. Example: +// chunks of 512KB and 1MB average below SMALL_IO and are good merge-reader candidates, while two +// 8MB chunks should stay on the raw random-access reader. size_t average_prefetch_range_size(const std::vector& ranges); -// Decide whether Arrow ReadAt() should be routed through MergeRangeFileReader for the current row -// group. This is intentionally stricter than the background warm-up path: +// Decide whether native data-page ReadAt() should be routed through MergeRangeFileReader for the +// current row group. This is intentionally stricter than the background warm-up path: // - no valid projected chunks -> nothing to merge; // - in-memory file readers already avoid remote random IO; // - average chunk size >= MergeRangeFileReader::SMALL_IO would make merged reading wasteful. bool should_use_merge_range_reader(const std::vector& ranges, size_t avg_io_size, bool is_in_memory_reader); +// HTTP range servers may reject an overlong range near EOF even after accepting the capability +// probe. Staging a bounded small HTTP object also turns all native page reads into memory copies. +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size); + } // namespace detail struct ParquetFileContext { - std::shared_ptr arrow_file; // Arrow wrapper for Doris FileReader - std::unique_ptr<::parquet::ParquetFileReader> file_reader; // Arrow Parquet file parser - std::shared_ptr<::parquet::FileMetaData> metadata; // footer metadata (RowGroup information) - const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema + // Native metadata, index, and data-page paths share Doris' FileReader without transferring + // ownership to an external metadata tree. + io::FileReaderSPtr native_file; + // Row-group-scoped view of native_file. Small projected chunks use MergeRangeFileReader; + // large chunks and in-memory files keep native_file. + io::FileReaderSPtr native_row_group_file; + io::IOContext* native_io_ctx = nullptr; + // V2-owned Thrift footer/schema used to construct native page/encoding readers. A cache hit is + // owned by native_meta_cache_handle; a miss without cache is owned by native_metadata_owner. + const NativeParquetMetadata* native_metadata = nullptr; + std::unique_ptr native_metadata_owner; + ObjLRUCache::CacheHandle native_meta_cache_handle; + int64_t native_footer_read_calls = 0; + int64_t native_footer_cache_hits = 0; + bool native_page_cache_enabled = false; + std::string native_page_cache_file_key; Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, - const io::FileDescription& file_description); - // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page - // decoding, so v2 caches the serialized bytes read inside these ranges and excludes - // footer/metadata reads that happen before registration. - void register_page_cache_ranges(std::vector ranges); + const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); + Status load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const; + Status load_native_page_indexes(int row_group_id, + const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, + int64_t* read_time = nullptr, + int64_t* parse_time = nullptr) const; // Best-effort asynchronous warm-up for Parquet column chunks. This only has an effect when // the underlying Doris file reader is a CachedRemoteFileReader; other readers keep the same // random-access behavior and simply skip prefetch. void prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx); - // Switch the active reader used by Arrow ReadAt() to v1's MergeRangeFileReader when the current - // row group's projected column chunks are small random IOs. This is the real v1-compatible - // prefetch path: subsequent Arrow page reads go through the merged reader instead of merely - // warming file cache in the background. Returns true when merge-range reading is active. - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size); - // Restore Arrow ReadAt() to the base Doris file reader and flush any active merge-reader - // counters. Row-group setup uses this before dictionary-page probes, because those probes are - // a separate pass over the column chunk from the later Arrow RecordReader data-page stream. + // Install the row-group-scoped MergeRangeFileReader on the native data-page path. Dictionary + // probes must run before this method because their native ReadAt order is independent of the + // sequential projected chunk ranges consumed by MergeRangeFileReader. + bool set_native_random_access_ranges(const std::vector& ranges, + size_t avg_io_size, RuntimeProfile* profile, + int64_t merge_read_slice_size); + const io::FileReaderSPtr& native_data_file() const { + return native_row_group_file != nullptr ? native_row_group_file : native_file; + } + // Restore native ReadAt() to the base Doris file reader and flush merge-reader counters. void reset_random_access_ranges(); ParquetPageCacheStats page_cache_stats() const; Status close(); }; -Status arrow_status_to_doris_status(const arrow::Status& status); - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index d41ff295ec4b3e..b56a57d419e0f3 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -18,6 +18,7 @@ #include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/parquet_statistics.h" +#include "runtime/file_scan_profile.h" namespace doris::format::parquet { @@ -26,9 +27,13 @@ void ParquetProfile::init(RuntimeProfile* profile) { return; } + file_scan_profile::ensure_hierarchy(profile); static const char* parquet_profile = "ParquetReader"; - ADD_TIMER_WITH_LEVEL(profile, parquet_profile, 1); + total_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, parquet_profile, file_scan_profile::FILE_READER, 1); + // Row-group counters are part of the long-standing ParquetReader profile contract. Keep them + // below the format node so profile parsers and operators can attribute pruning to Parquet. filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsFiltered", TUnit::UNIT, parquet_profile, 1); filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -55,10 +60,16 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::BYTES, parquet_profile, 1); selected_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRows", TUnit::UNIT, parquet_profile, 1); + // Keep every Parquet scan metric below the format node: profile consumers extract that + // subtree and otherwise silently lose this counter even though filtering happened. rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByConjunct", TUnit::UNIT, parquet_profile, 1); total_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); + dense_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DenseBatches", TUnit::UNIT, parquet_profile, 1); + selected_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedBatches", TUnit::UNIT, + parquet_profile, 1); empty_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptySelectionBatches", TUnit::UNIT, parquet_profile, 1); range_gap_skipped_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RangeGapSkippedRows", @@ -69,12 +80,30 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); reader_select_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSelectRows", TUnit::UNIT, parquet_profile, 1); - arrow_read_records_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowReadRecordsTime", parquet_profile, 1); + level_only_read_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlyReadTime", parquet_profile, 1); + level_only_skip_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlySkipTime", parquet_profile, 1); materialization_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); + hybrid_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionBatches", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionRanges", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_null_fallback_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "HybridSelectionNullFallbackBatches", TUnit::UNIT, parquet_profile, 1); + native_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativeReadCalls", TUnit::UNIT, + parquet_profile, 1); + native_page_fragments = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativePageFragments", + TUnit::UNIT, parquet_profile, 1); + page_crossing_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCrossingBatches", + TUnit::UNIT, parquet_profile, 1); + nested_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NestedBatches", TUnit::UNIT, parquet_profile, 1); lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", TUnit::UNIT, parquet_profile, 1); + // Format-specific counters stay below ParquetReader so subtree consumers cannot silently + // attribute them to a different file format initialized on the same RuntimeProfile. filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, parquet_profile, 1); raw_rows_read = @@ -86,7 +115,8 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, parquet_profile, 1); - page_index_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, 1); + page_index_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, + parquet_profile, 1); page_index_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexFilterTime", parquet_profile, 1); read_page_index_time = @@ -101,8 +131,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); row_group_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); - file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); - file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); + file_footer_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", + TUnit::UNIT, parquet_profile, 1); + file_footer_hit_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, + parquet_profile, 1); decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1); @@ -137,6 +169,16 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); predicate_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateFilterTime", parquet_profile, 1); + predicate_compaction_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateCompactionTime", parquet_profile, 1); + predicate_compaction_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionBytes", + TUnit::BYTES, parquet_profile, 1); + predicate_compaction_count = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionCount", + TUnit::UNIT, parquet_profile, 1); + fixed_width_predicate_direct_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "FixedWidthPredicateDirectBatches", TUnit::UNIT, parquet_profile, 1); + fixed_width_predicate_direct_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "FixedWidthPredicateDirectRows", TUnit::UNIT, parquet_profile, 1); dict_filter_rewrite_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterRewriteTime", parquet_profile, 1); dict_filter_expr_rewrite_time = @@ -155,7 +197,6 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); rows_filtered_by_dict_filter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByDictFilter", TUnit::UNIT, parquet_profile, 1); - convert_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ConvertTime", parquet_profile, 1); bloom_filter_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "BloomFilterReadTime", parquet_profile, 1); } @@ -174,12 +215,42 @@ void ParquetProfile::update_pruning_stats(const ParquetPruningStats& pruning_sta COUNTER_UPDATE(total_row_groups, pruning_stats.total_row_groups); COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); + COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); + COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); + COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); + COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); + COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); + COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); + COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); + COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); + COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); +} + +void ParquetProfile::update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const { + const int64_t filtered = selected ? 0 : 1; + COUNTER_UPDATE(filtered_row_groups, filtered); + COUNTER_UPDATE(filtered_row_groups_by_dictionary, + pruning_stats.filtered_row_groups_by_dictionary); + COUNTER_UPDATE(filtered_row_groups_by_bloom_filter, + pruning_stats.filtered_row_groups_by_bloom_filter); + COUNTER_UPDATE(filtered_row_groups_by_page_index, + pruning_stats.filtered_row_groups_by_page_index); + // Initial footer planning counts every surviving candidate as readable. Lazy probes correct + // that estimate only when a later dictionary, Bloom, or page-index check removes the group. + COUNTER_UPDATE(to_read_row_groups, -filtered); + COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); + COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); @@ -197,8 +268,35 @@ ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { .reader_read_rows = reader_read_rows, .reader_skip_rows = reader_skip_rows, .reader_select_rows = reader_select_rows, - .arrow_read_records_time = arrow_read_records_time, + .level_only_read_time = level_only_read_time, + .level_only_skip_time = level_only_skip_time, .materialization_time = materialization_time, + .hybrid_selection_batches = hybrid_selection_batches, + .hybrid_selection_ranges = hybrid_selection_ranges, + .hybrid_selection_null_fallback_batches = hybrid_selection_null_fallback_batches, + .decompress_time = decompress_time, + .decompress_count = decompress_cnt, + .decode_header_time = decode_header_time, + .decode_value_time = decode_value_time, + .decode_dictionary_time = decode_dict_time, + .decode_level_time = decode_level_time, + .decode_null_map_time = decode_null_map_time, + .page_index_read_calls = page_index_read_calls, + .skip_page_header_count = skip_page_header_num, + .parse_page_header_count = parse_page_header_num, + .read_page_header_time = read_page_header_time, + .page_read_count = page_read_counter, + .page_cache_write_count = page_cache_write_counter, + .page_cache_compressed_write_count = page_cache_compressed_write_counter, + .page_cache_decompressed_write_count = page_cache_decompressed_write_counter, + .page_cache_hit_count = page_cache_hit_counter, + .page_cache_miss_count = page_cache_missing_counter, + .page_cache_compressed_hit_count = page_cache_compressed_hit_counter, + .page_cache_decompressed_hit_count = page_cache_decompressed_hit_counter, + .native_read_calls = native_read_calls, + .native_page_fragments = native_page_fragments, + .page_crossing_batches = page_crossing_batches, + .nested_batches = nested_batches, }; } @@ -209,10 +307,17 @@ ParquetScanProfile ParquetProfile::scan_profile() const { .rows_filtered_by_conjunct = rows_filtered_by_conjunct, .lazy_read_filtered_rows = lazy_read_filtered_rows, .total_batches = total_batches, + .dense_batches = dense_batches, + .selected_batches = selected_batches, .empty_selection_batches = empty_selection_batches, .range_gap_skipped_rows = range_gap_skipped_rows, .column_read_time = column_read_time, .predicate_filter_time = predicate_filter_time, + .predicate_compaction_time = predicate_compaction_time, + .predicate_compaction_bytes = predicate_compaction_bytes, + .predicate_compaction_count = predicate_compaction_count, + .fixed_width_predicate_direct_batches = fixed_width_predicate_direct_batches, + .fixed_width_predicate_direct_rows = fixed_width_predicate_direct_rows, .dict_filter_rewrite_time = dict_filter_rewrite_time, .dict_filter_expr_rewrite_time = dict_filter_expr_rewrite_time, .dict_filter_read_dict_time = dict_filter_read_dict_time, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 27f9818f4a0dcd..170f14b56c5d14 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -31,27 +31,65 @@ struct ParquetPageSkipProfile { // ============================================================================ // ============================================================================ struct ParquetColumnReaderProfile { - RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() - RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() - RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() - RuntimeProfile::Counter* arrow_read_records_time = nullptr; // Arrow RecordReader time (ns) - RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() + RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() + RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() + // COUNT(nullable_col) shape-only path; ordinary scans keep both counters zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; + RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; + // Native page/encoding reader internals. These counters keep page IO, decompression, levels, + // value decode and conversion attributable to separate stages. + RuntimeProfile::Counter* decompress_time = nullptr; + RuntimeProfile::Counter* decompress_count = nullptr; + RuntimeProfile::Counter* decode_header_time = nullptr; + RuntimeProfile::Counter* decode_value_time = nullptr; + RuntimeProfile::Counter* decode_dictionary_time = nullptr; + RuntimeProfile::Counter* decode_level_time = nullptr; + RuntimeProfile::Counter* decode_null_map_time = nullptr; + RuntimeProfile::Counter* page_index_read_calls = nullptr; + RuntimeProfile::Counter* skip_page_header_count = nullptr; + RuntimeProfile::Counter* parse_page_header_count = nullptr; + RuntimeProfile::Counter* read_page_header_time = nullptr; + RuntimeProfile::Counter* page_read_count = nullptr; + RuntimeProfile::Counter* page_cache_write_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_miss_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_hit_count = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; // native column-reader calls + RuntimeProfile::Counter* native_page_fragments = nullptr; // page-bounded read fragments + RuntimeProfile::Counter* page_crossing_batches = nullptr; // batches spanning multiple pages + RuntimeProfile::Counter* nested_batches = nullptr; // complex-column read batches }; // ============================================================================ // ============================================================================ struct ParquetScanProfile { - RuntimeProfile::Counter* raw_rows_read = nullptr; // raw rows read from RecordReader + RuntimeProfile::Counter* raw_rows_read = nullptr; // logical rows consumed before filtering RuntimeProfile::Counter* selected_rows = nullptr; // rows selected after conjunct filtering RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; // rows filtered by conjuncts RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; // rows avoided by late materialization RuntimeProfile::Counter* total_batches = nullptr; // total batch count + RuntimeProfile::Counter* dense_batches = nullptr; // batches retaining every physical row + RuntimeProfile::Counter* selected_batches = + nullptr; // non-empty batches compacted by predicates RuntimeProfile::Counter* empty_selection_batches = nullptr; // empty batches after full filtering RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; // rows skipped by range gaps RuntimeProfile::Counter* column_read_time = nullptr; // column read time (ns) RuntimeProfile::Counter* predicate_filter_time = nullptr; // predicate filter time (ns) + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; // dictionary rewrite time (ns) RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; // expression/residual rewrite time (ns) @@ -72,11 +110,15 @@ struct ParquetScanProfile { struct ParquetProfile { void init(RuntimeProfile* profile); void update_pruning_stats(const ParquetPruningStats& pruning_stats) const; + void update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const; ParquetPageSkipProfile page_skip_profile() const; ParquetColumnReaderProfile column_reader_profile() const; ParquetScanProfile scan_profile() const; + RuntimeProfile::Counter* total_time = nullptr; + RuntimeProfile::Counter* filtered_row_groups = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_min_max = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_dictionary = nullptr; @@ -95,6 +137,8 @@ struct ParquetProfile { RuntimeProfile::Counter* selected_rows = nullptr; RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; RuntimeProfile::Counter* total_batches = nullptr; + RuntimeProfile::Counter* dense_batches = nullptr; + RuntimeProfile::Counter* selected_batches = nullptr; RuntimeProfile::Counter* empty_selection_batches = nullptr; RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; @@ -102,8 +146,17 @@ struct ParquetProfile { RuntimeProfile::Counter* reader_read_rows = nullptr; RuntimeProfile::Counter* reader_skip_rows = nullptr; RuntimeProfile::Counter* reader_select_rows = nullptr; - RuntimeProfile::Counter* arrow_read_records_time = nullptr; + // COUNT(nullable_col) shape-only path; ordinary scans keep these zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; + RuntimeProfile::Counter* native_page_fragments = nullptr; + RuntimeProfile::Counter* page_crossing_batches = nullptr; + RuntimeProfile::Counter* nested_batches = nullptr; RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; RuntimeProfile::Counter* filtered_bytes = nullptr; @@ -147,6 +200,11 @@ struct ParquetProfile { RuntimeProfile::Counter* parse_page_header_num = nullptr; RuntimeProfile::Counter* predicate_filter_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* fixed_width_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_read_dict_time = nullptr; @@ -156,7 +214,6 @@ struct ParquetProfile { RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; RuntimeProfile::Counter* dict_filter_read_failures = nullptr; RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; - RuntimeProfile::Counter* convert_time = nullptr; RuntimeProfile::Counter* bloom_filter_read_time = nullptr; }; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 753b3628bfa19b..4a80bc0403df8d 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -39,7 +39,7 @@ #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/parquet_statistics.h" -#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/count_column_reader.h" #include "io/io_common.h" #include "runtime/runtime_state.h" @@ -57,34 +57,26 @@ struct ParquetReaderScanState { bool enable_strict_mode = false; }; -int64_t column_chunk_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); -} - -void collect_all_leaf_column_ids(const ParquetColumnSchema& column_schema, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); +Status validate_all_projected_leaves_supported(const ParquetColumnSchema& column_schema) { if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { - if (column_schema.leaf_column_id >= 0) { - leaf_column_ids->insert(column_schema.leaf_column_id); + if (!column_schema.type_descriptor.unsupported_reason.empty()) { + return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, + column_schema.type_descriptor.unsupported_reason); } - return; + return Status::OK(); } for (const auto& child : column_schema.children) { DORIS_CHECK(child != nullptr); - collect_all_leaf_column_ids(*child, leaf_column_ids); + RETURN_IF_ERROR(validate_all_projected_leaves_supported(*child)); } + return Status::OK(); } -void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex& projection, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - if (projection.project_all_children || projection.children.empty()) { - collect_all_leaf_column_ids(column_schema, leaf_column_ids); - return; +Status validate_projected_leaves_supported(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex& projection) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE || + projection.project_all_children || projection.children.empty()) { + return validate_all_projected_leaves_supported(column_schema); } for (const auto& child_projection : projection.children) { const auto child_it = @@ -92,57 +84,35 @@ void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, return child_schema->local_id == child_projection.local_id(); }); DORIS_CHECK(child_it != column_schema.children.end()); - collect_projected_leaf_column_ids(**child_it, child_projection, leaf_column_ids); + RETURN_IF_ERROR(validate_projected_leaves_supported(**child_it, child_projection)); } + return Status::OK(); } -void collect_request_leaf_column_ids( +Status validate_requested_columns_supported( const std::vector>& file_schema, - const format::FileScanRequest& request, std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - auto collect_scan_column = [&](const format::LocalColumnIndex& projection) { + const format::FileScanRequest& request) { + // Validate the projected native-schema leaves before pruning: checking a physical carrier at + // a later read site can let an unsupported logical type silently pass when every row is pruned. + auto validate_scan_column = [&](const format::LocalColumnIndex& projection) -> Status { const auto local_id = projection.local_id(); if (local_id == format::ROW_POSITION_COLUMN_ID || local_id == format::GLOBAL_ROWID_COLUMN_ID) { - return; + return Status::OK(); } DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); DORIS_CHECK(file_schema[local_id] != nullptr); - collect_projected_leaf_column_ids(*file_schema[local_id], projection, leaf_column_ids); + return validate_projected_leaves_supported(*file_schema[local_id], projection); }; for (const auto& column : request.predicate_columns) { - collect_scan_column(column); + RETURN_IF_ERROR(validate_scan_column(column)); } for (const auto& column : request.non_predicate_columns) { - collect_scan_column(column); - } -} - -std::vector build_page_cache_ranges( - const ::parquet::FileMetaData& metadata, - const std::vector>& file_schema, - const format::FileScanRequest& request, const RowGroupScanPlan& row_group_plan) { - std::unordered_set leaf_column_ids; - collect_request_leaf_column_ids(file_schema, request, &leaf_column_ids); - std::vector ranges; - ranges.reserve(row_group_plan.row_groups.size() * leaf_column_ids.size()); - for (const auto& row_group_plan_item : row_group_plan.row_groups) { - auto row_group_metadata = metadata.RowGroup(row_group_plan_item.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - for (const auto leaf_column_id : leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_chunk_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); - DORIS_CHECK(offset >= 0); - DORIS_CHECK(size >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); - } + if (!request.is_count_star_placeholder(column.column_id())) { + RETURN_IF_ERROR(validate_scan_column(column)); } } - return ranges; + return Status::OK(); } const ParquetColumnSchema& projected_root_schema( @@ -155,11 +125,10 @@ const ParquetColumnSchema& projected_root_schema( } int64_t count_loaded_non_null_values(const ParquetColumnSchema& root_schema, - const ParquetColumnReader& shape_reader, - int64_t expected_rows) { - const auto& def_levels = shape_reader.nested_definition_levels(); - const auto& rep_levels = shape_reader.nested_repetition_levels(); - const int64_t levels_written = shape_reader.nested_levels_written(); + const CountColumnReader& shape_reader, int64_t expected_rows) { + const auto& def_levels = shape_reader.definition_levels(); + const auto& rep_levels = shape_reader.repetition_levels(); + const int64_t levels_written = shape_reader.levels_written(); DORIS_CHECK(levels_written >= expected_rows); if (root_schema.max_repetition_level == 0) { DORIS_CHECK(levels_written == expected_rows); @@ -207,7 +176,7 @@ int timestamp_tz_scale(const ParquetTypeDescriptor& type_descriptor) { bool should_map_to_timestamp_tz(const ParquetColumnSchema& column_schema) { const auto& type_descriptor = column_schema.type_descriptor; - return type_descriptor.physical_type == ::parquet::Type::INT96 || + return type_descriptor.physical_type == tparquet::Type::INT96 || (type_descriptor.is_timestamp && type_descriptor.timestamp_is_adjusted_to_utc); } @@ -288,10 +257,9 @@ static Status find_projected_minmax_leaf(const ParquetColumnSchema& column_schem } static Status validate_minmax_aggregate_statistics(const ParquetColumnSchema& column_schema) { - DORIS_CHECK(column_schema.descriptor != nullptr); - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + switch (column_schema.type_descriptor.physical_type) { + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: // Arrow 17 does not expose Parquet's min/max exactness flags. Binary statistics may be // truncated bounds rather than values present in the file, so they are safe for pruning // but cannot be returned as exact aggregate results. @@ -328,14 +296,17 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _enable_mapping_varbinary(enable_mapping_varbinary) {} ParquetReader::~ParquetReader() = default; Status ParquetReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_parquet_profile.total_time); if (_io_ctx != nullptr && _io_ctx->should_stop) { return Status::EndOfFile("stop"); } @@ -356,6 +327,7 @@ Status ParquetReader::init(RuntimeState* state) { _state->enable_strict_mode = state->enable_strict_mode(); _state->scheduler.set_timezone(&state->timezone_obj()); _state->scheduler.set_enable_strict_mode(_state->enable_strict_mode); + _state->scheduler.set_runtime_state(state); } int64_t merge_read_slice_size = -1; if (state != nullptr && state->query_options().__isset.merge_read_slice_size) { @@ -363,16 +335,30 @@ Status ParquetReader::init(RuntimeState* state) { } _state->scheduler.set_merge_read_options(_profile, merge_read_slice_size); _state->scheduler.set_batch_size(_batch_size); - // Open parquet file and parse metadata to get file schema. - RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description)); + // Opening the file parses the footer before any row group can be scheduled. Keep this timer + // around the whole operation so footer/cache latency cannot disappear from a slow profile. + { + SCOPED_TIMER(_parquet_profile.parse_footer_time); + RETURN_IF_ERROR(_state->file_context.open( + _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, + _enable_mapping_timestamp_tz, _enable_mapping_varbinary)); + } + if (_profile != nullptr) { + COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, + _state->file_context.native_footer_read_calls); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, + _state->file_context.native_footer_cache_hits); + } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier - RETURN_IF_ERROR( - build_parquet_column_schema(*_state->file_context.schema, &_state->file_schema)); - if (_enable_mapping_timestamp_tz) { - for (auto& column_schema : _state->file_schema) { - apply_timestamp_tz_mapping(column_schema.get()); + { + SCOPED_TIMER(_parquet_profile.parse_meta_time); + RETURN_IF_ERROR(build_parquet_column_schema(_state->file_context.native_metadata->schema(), + &_state->file_schema)); + if (_enable_mapping_timestamp_tz) { + for (auto& column_schema : _state->file_schema) { + apply_timestamp_tz_mapping(column_schema.get()); + } } } return Status::OK(); @@ -386,11 +372,12 @@ void ParquetReader::set_batch_size(size_t batch_size) { } Status ParquetReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_parquet_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } file_schema->clear(); - if (_state == nullptr || _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } @@ -413,8 +400,8 @@ std::unique_ptr ParquetReader::create_column_mapper( } Status ParquetReader::open(std::shared_ptr request) { - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + SCOPED_TIMER(_parquet_profile.total_time); + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } auto request_snapshot = request; @@ -454,6 +441,12 @@ Status ParquetReader::open(std::shared_ptr request) { DORIS_CHECK(local_id >= 0 && local_id < num_fields); } + // Reject requested unsupported logical leaves before row-group statistics, dictionaries, + // bloom filters or page indexes inspect their physical fallback type. For example, a predicate + // on TIME_MILLIS must fail here even when its INT32 statistics would prune every row group; + // otherwise the same unsupported SELECT could fail or silently succeed depending on data. + RETURN_IF_ERROR(validate_requested_columns_supported(_state->file_schema, *request_snapshot)); + RowGroupScanPlan row_group_plan; ParquetScanRange scan_range; scan_range.start_offset = _file_description->range_start_offset; @@ -461,19 +454,20 @@ Status ParquetReader::open(std::shared_ptr request) { scan_range.file_size = _file_description->file_size; // Get selected ranges in row groups according to metadata (Row-Group level index and Page Index including Zonemap, Dictionary, Bloom Filter). RETURN_IF_ERROR(plan_parquet_row_groups( - *_state->file_context.metadata, _state->file_context.file_reader.get(), - _state->file_schema, *request_snapshot, scan_range, _state->enable_bloom_filter, - &row_group_plan, _state->timezone, _state->runtime_state)); + *_state->file_context.native_metadata, _state->file_schema, *request_snapshot, + scan_range, _state->enable_bloom_filter, &row_group_plan, _state->timezone, + _state->runtime_state, &_state->file_context, + _parquet_profile.column_reader_profile())); if (_profile != nullptr) { _parquet_profile.update_pruning_stats(row_group_plan.pruning_stats); } - if (_state->enable_page_cache) { - _state->file_context.register_page_cache_ranges( - build_page_cache_ranges(*_state->file_context.metadata, _state->file_schema, - *request_snapshot, row_group_plan)); - } + // Native page readers admit exact validated page payloads to cache. Do not pre-register whole + // column chunks here: footer offsets are untrusted and this obsolete range map is not consumed. _state->scan_plan = row_group_plan; _state->scheduler.set_page_skip_profile(_parquet_profile.page_skip_profile()); + if (_profile != nullptr) { + _state->scheduler.set_pruning_profile(&_parquet_profile); + } _state->scheduler.set_global_rowid_context(_global_rowid_context); _state->scheduler.set_scan_profile(_parquet_profile.scan_profile()); _state->scheduler.set_plan(std::move(row_group_plan)); @@ -482,8 +476,8 @@ Status ParquetReader::open(std::shared_ptr request) { } Status ParquetReader::get_block(Block* file_block, size_t* rows, bool* eof) { - if (_state == nullptr || _state->file_context.file_reader == nullptr || - _state->file_context.schema == nullptr) { + SCOPED_TIMER(_parquet_profile.total_time); + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } *rows = 0; @@ -582,9 +576,9 @@ int64_t ParquetReader::get_total_rows() const { Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_parquet_profile.total_time); DORIS_CHECK(result != nullptr); - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } if (_should_stop()) { @@ -598,12 +592,32 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r request.agg_type); } + // Aggregate pushdown bypasses the scheduler but still requires the exact pruned row-group set. + // Finish lazy remote probes here; normal scans keep them at current-row-group granularity. + RETURN_IF_ERROR(finalize_parquet_row_group_plans( + *_state->file_context.native_metadata, _state->file_schema, *_request, + _state->enable_bloom_filter, &_state->scan_plan, _state->timezone, + _state->runtime_state, &_state->file_context, _parquet_profile.column_reader_profile(), + _profile == nullptr ? nullptr : &_parquet_profile)); + + for (const auto& aggregate_column : request.columns) { + const auto local_id = aggregate_column.projection.local_id(); + if (local_id < 0 || local_id >= static_cast(_state->file_schema.size())) { + return Status::InvalidArgument("Invalid parquet aggregate column id {}", local_id); + } + DORIS_CHECK(_state->file_schema[local_id] != nullptr); + // Aggregate pushdown can return directly from footer statistics without constructing a + // column reader. Validate first so MIN/MAX(TIME_MILLIS), or an all-pruned COUNT request, + // cannot expose the physical INT32 fallback as a supported logical value. + RETURN_IF_ERROR(validate_projected_leaves_supported(*_state->file_schema[local_id], + aggregate_column.projection)); + } + // Aggregate row count in all selected row groups. For MIN/MAX aggregate, this is used to determine whether there is no row group selected. for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - result->count += row_group_metadata->num_rows(); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + result->count += row_group_metadata.num_rows; } if (request.agg_type == TPushAggOp::type::COUNT) { if (request.columns.empty()) { @@ -614,33 +628,26 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } const auto& count_projection = request.columns[0].projection; const auto& root_schema = projected_root_schema(_state->file_schema, count_projection); + // A required primitive COUNT(col) still carries its projection so the unsupported-type + // validation above cannot be bypassed. Once validated, its definition level proves that + // every selected row is non-NULL, so preserve the already-computed footer row count and + // avoid reading definition levels merely to rediscover COUNT(col) == COUNT(*). Complex + // roots continue through the shape reader because their count semantics and read-row + // accounting are derived from nested levels. + if (root_schema.kind == ParquetColumnSchemaKind::PRIMITIVE && + root_schema.max_definition_level == 0) { + return Status::OK(); + } result->count = 0; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - std::shared_ptr<::parquet::RowGroupReader> row_group; - try { - row_group = _state->file_context.file_reader->RowGroup(row_group_plan.row_group_id); - } catch (const ::parquet::ParquetException& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } catch (const std::exception& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } - - ParquetColumnReaderFactory column_reader_factory( - row_group, _state->file_context.schema->num_columns(), - &row_group_plan.page_skip_plans, _parquet_profile.page_skip_profile(), - _state->timezone, _state->enable_strict_mode, - _parquet_profile.scan_profile().column_reader_profile); - std::unique_ptr shape_reader; - RETURN_IF_ERROR(column_reader_factory.create_count_shape_reader( - root_schema, &count_projection, &shape_reader)); + std::unique_ptr shape_reader; + RETURN_IF_ERROR(CountColumnReader::create( + _state->file_context.native_data_file(), _state->file_context.native_metadata, + row_group_plan.row_group_id, root_schema, &count_projection, + _state->file_context.native_io_ctx, + _state->file_context.native_page_cache_enabled, + _state->file_context.native_page_cache_file_key, + _parquet_profile.scan_profile().column_reader_profile, &shape_reader)); DORIS_CHECK(shape_reader != nullptr); int64_t row_group_cursor = 0; @@ -654,18 +661,19 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r while (range_rows_read < selected_range.length) { const int64_t batch_rows = std::min(_batch_size, selected_range.length - range_rows_read); - // COUNT(col) only needs the top-level NULL state. The shape reader loads - // def/rep levels from one representative leaf and does not build value_indices - // or values_column. MAP chooses the key leaf; ARRAY/STRUCT may choose a string - // leaf, but the levels-only protocol still avoids Doris-side string - // materialization for that leaf. + int64_t rows_read = 0; RETURN_IF_ERROR(_stop_status_if_requested( - shape_reader->load_nested_levels_batch(batch_rows))); - _record_scan_rows(batch_rows); + shape_reader->read_levels(batch_rows, &rows_read))); + if (rows_read != batch_rows) { + return Status::Corruption( + "Parquet COUNT reader returned {} rows, expected {}", rows_read, + batch_rows); + } + _record_scan_rows(rows_read); result->count += - count_loaded_non_null_values(root_schema, *shape_reader, batch_rows); - range_rows_read += batch_rows; - row_group_cursor += batch_rows; + count_loaded_non_null_values(root_schema, *shape_reader, rows_read); + range_rows_read += rows_read; + row_group_cursor += rows_read; } } } @@ -692,13 +700,30 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r auto& aggregate_column = result->columns[request_column_idx]; aggregate_column.projection = request.columns[request_column_idx].projection; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - auto column_chunk = row_group_metadata->ColumnChunk(leaf_schema->leaf_column_id); - DORIS_CHECK(column_chunk != nullptr); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + DORIS_CHECK(leaf_schema->leaf_column_id >= 0 && + leaf_schema->leaf_column_id < + static_cast(row_group_metadata.columns.size())); + const auto& column_chunk = row_group_metadata.columns[leaf_schema->leaf_column_id]; + DORIS_CHECK(column_chunk.__isset.meta_data); + const auto& column_metadata = column_chunk.meta_data; + std::optional safe_statistics; + if (column_metadata.__isset.statistics && + detail::can_use_native_footer_min_max( + leaf_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order( + _state->file_context.native_metadata->to_thrift(), + leaf_schema->leaf_column_id))) { + safe_statistics = detail::sanitize_native_footer_statistics( + leaf_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order( + _state->file_context.native_metadata->to_thrift(), + leaf_schema->leaf_column_id)); + } const auto statistics = ParquetStatisticsUtils::TransformColumnStatistics( - *leaf_schema, column_chunk->statistics(), _state->timezone); + *leaf_schema, safe_statistics.has_value() ? &*safe_statistics : nullptr, + column_metadata.num_values, _state->timezone); if (!statistics.has_min_max) { return Status::NotSupported("Missing parquet min/max statistics for column {}", leaf_schema->name); @@ -720,7 +745,9 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } Status ParquetReader::close() { + SCOPED_TIMER(_parquet_profile.total_time); if (_state != nullptr) { + _state->scheduler.close(); _sync_page_cache_profile(); RETURN_IF_ERROR(_state->file_context.close()); } diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 6c8e88cc27a9b6..da6135b81ae838 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -46,7 +46,7 @@ class ParquetReader : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); ~ParquetReader() override; Status init(RuntimeState* state) override; @@ -89,6 +89,7 @@ class ParquetReader : public format::FileReader { std::optional _global_rowid_context; // global RowId context size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ + bool _enable_mapping_varbinary = false; // whether raw BYTE_ARRAY is mapped to VARBINARY }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index a7602cd11b27d0..b92a3c27054ad1 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -15,11 +15,11 @@ #include "format_v2/parquet/parquet_scan.h" -#include - #include +#include #include #include +#include #include #include #include @@ -35,37 +35,93 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/global_rowid_column_reader.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native_column_reader.h" +#include "format_v2/parquet/reader/row_position_column_reader.h" +#include "util/defer_op.h" +#include "util/time.h" namespace doris::format::parquet { -namespace { +namespace detail { + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats) { + if (std::ranges::any_of(positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return positions; + } + auto ordered = positions; + std::stable_sort(ordered.begin(), ordered.end(), [&](size_t left, size_t right) { + const auto score = [&](size_t position) { + const auto& sample = stats.at(position); + return sample.cost_per_input_row_ns / std::max(1.0 - sample.survival_ratio, 0.01); + }; + return score(left) < score(right); + }); + return ordered; +} + +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability) { + if (std::ranges::any_of(ordered_positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return ordered_positions; + } + std::vector result; + double reach_probability = 1; + for (const size_t position : ordered_positions) { + if (!result.empty() && reach_probability < minimum_reach_probability) { + break; + } + result.push_back(position); + reach_probability *= stats.at(position).survival_ratio; + } + return result; +} -int64_t column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence) { + constexpr size_t WARMUP_SAMPLES = 8; + constexpr size_t STEADY_STATE_INTERVAL = 16; + return samples < WARMUP_SAMPLES || batch_sequence % STEADY_STATE_INTERVAL == 0; } -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; +} // namespace detail + +namespace { + +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( + const format::FileScanRequest& request); + +bool is_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; } -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; +bool is_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; } -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; +bool is_data_page_type(tparquet::PageType::type page_type) { + return page_type == tparquet::PageType::DATA_PAGE || + page_type == tparquet::PageType::DATA_PAGE_V2; } -bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { +bool is_fully_dictionary_encoded_chunk(const tparquet::ColumnMetaData& column_metadata) { + if (!column_metadata.__isset.dictionary_page_offset || + column_metadata.dictionary_page_offset < 0) { return false; } - const auto& encoding_stats = column_metadata.encoding_stats(); + const auto& encoding_stats = column_metadata.encoding_stats; if (!encoding_stats.empty()) { bool has_dictionary_data_page = false; for (const auto& encoding_stat : encoding_stats) { @@ -81,7 +137,7 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { + for (const auto encoding : column_metadata.encodings) { if (is_dictionary_data_encoding(encoding)) { has_dictionary_encoding = true; continue; @@ -94,19 +150,39 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool supports_row_level_dictionary_filter(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr || + const tparquet::ColumnMetaData& column_metadata) { + if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || column_schema.type == nullptr || column_schema.max_repetition_level > 0) { return false; } - if (!column_schema.type_descriptor.is_string_like || - column_metadata.type() != ::parquet::Type::BYTE_ARRAY) { + bool is_supported_physical_type = false; + switch (column_metadata.type) { + case tparquet::Type::BYTE_ARRAY: + is_supported_physical_type = column_schema.type_descriptor.is_string_like; + break; + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + is_supported_physical_type = true; + break; + case tparquet::Type::BOOLEAN: + // Parquet booleans are PLAIN encoded and cannot have a dictionary page. + break; + } + if (!is_supported_physical_type) { return false; } - // Row-level dictionary filtering consumes dictionary ids from DATA_PAGE payloads. It is exact - // only when every data page is dictionary encoded. Mixed dictionary/plain chunks are left on - // the normal decoded-value path, matching the safety rule used by StarRocks and Doris v1. + if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_VARBINARY) { + // A table STRING predicate can be rewritten through a raw VARBINARY file slot. Evaluating + // it on dictionary Fields before the mapping expression is neither type-safe nor exact. + return false; + } + // The row filter consumes dictionary ids rather than decoded values, so a plain data page + // cannot resume this reader without changing its output domain. Keep mixed chunks on the + // normal decoded-value path to preserve one representation for the complete column chunk. return is_fully_dictionary_encoded_chunk(column_metadata); } @@ -143,49 +219,57 @@ void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, } } -bool is_row_group_outside_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, int row_group_idx) { - if (scan_range.size < 0) { - return false; - } - const int64_t range_start_offset = scan_range.start_offset; - const int64_t range_end_offset = range_start_offset + scan_range.size; - DORIS_CHECK(range_start_offset >= 0); - DORIS_CHECK(range_end_offset >= range_start_offset); - if (range_start_offset == 0 && - (scan_range.file_size < 0 || range_end_offset >= scan_range.file_size)) { - return false; - } - - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - DORIS_CHECK(row_group_metadata->num_columns() > 0); - const auto first_column = row_group_metadata->ColumnChunk(0); - const auto last_column = row_group_metadata->ColumnChunk(row_group_metadata->num_columns() - 1); - DORIS_CHECK(first_column != nullptr); - DORIS_CHECK(last_column != nullptr); - const int64_t row_group_start_offset = column_start_offset(*first_column); - const int64_t row_group_end_offset = - column_start_offset(*last_column) + last_column->total_compressed_size(); - const int64_t row_group_mid_offset = - row_group_start_offset + (row_group_end_offset - row_group_start_offset) / 2; - return row_group_mid_offset < range_start_offset || row_group_mid_offset >= range_end_offset; -} - std::vector request_scan_columns(const format::FileScanRequest& request) { std::vector scan_columns; scan_columns.reserve(request.predicate_columns.size() + request.non_predicate_columns.size()); scan_columns.insert(scan_columns.end(), request.predicate_columns.begin(), request.predicate_columns.end()); - scan_columns.insert(scan_columns.end(), request.non_predicate_columns.begin(), - request.non_predicate_columns.end()); + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + scan_columns.push_back(column); + } + } return scan_columns; } -std::vector build_row_group_prefetch_ranges( - const ::parquet::FileMetaData& metadata, +std::vector physical_non_predicate_columns( + const format::FileScanRequest& request) { + std::vector columns; + columns.reserve(request.non_predicate_columns.size()); + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + columns.push_back(column); + } + } + return columns; +} + +void materialize_count_star_placeholders(const format::FileScanRequest& request, size_t rows, + Block* file_block) { + DORIS_CHECK(file_block != nullptr); + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + continue; + } + const auto block_position = request.local_positions.at(column.column_id()).value(); + auto placeholder = file_block->get_by_position(block_position).column->assert_mutable(); + DCHECK(placeholder->empty()); + placeholder->insert_many_defaults(rows); + file_block->replace_by_position(block_position, std::move(placeholder)); + } +} + +} // namespace + +namespace detail { + +Status build_native_prefetch_ranges( + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, - const std::vector& scan_columns, int row_group_idx) { + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); std::unordered_set leaf_column_ids; for (const auto& projection : scan_columns) { const auto local_id = projection.local_id(); @@ -193,92 +277,233 @@ std::vector build_row_group_prefetch_ranges( local_id == format::GLOBAL_ROWID_COLUMN_ID) { continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return Status::Corruption("Invalid Parquet projected column id {}", local_id); + } // Prefetch and merge-reader ranges must be physical leaf chunks, not Doris logical slots. // Example: for a struct column s, projecting only s.a should include only // the Parquet leaf chunk of a. Projecting the whole struct includes both a and b. collect_projected_leaf_column_ids(*file_schema[local_id], projection, &leaf_column_ids); } - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group index {}", row_group_idx); + } + const auto& row_group_metadata = metadata.row_groups[row_group_idx]; std::vector ordered_leaf_column_ids(leaf_column_ids.begin(), leaf_column_ids.end()); std::ranges::sort(ordered_leaf_column_ids); - std::vector ranges; - ranges.reserve(ordered_leaf_column_ids.size()); + ranges->reserve(ordered_leaf_column_ids.size()); for (const auto leaf_column_id : ordered_leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); - DORIS_CHECK(offset >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(row_group_metadata.columns.size())) { + return Status::Corruption("Invalid Parquet leaf column id {}", leaf_column_id); + } + const auto& chunk = row_group_metadata.columns[leaf_column_id]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet leaf column {} has no chunk metadata", + leaf_column_id); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range(chunk.meta_data, file_size, + parquet_816_padding, &chunk_range)); + if (chunk_range.length > 0) { + if (chunk_range.offset > static_cast(std::numeric_limits::max()) || + chunk_range.length > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet column chunk range exceeds int64 coordinates"); + } + // Prefetch must use the same checked chunk extent as the decoder, including the + // PARQUET-816 compatibility padding, so warm-up cannot target different bytes. + ranges->push_back( + ParquetPageCacheRange {.offset = static_cast(chunk_range.offset), + .size = static_cast(chunk_range.length)}); } } - return ranges; + return Status::OK(); } -Status select_row_groups_by_scan_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, - std::vector* row_group_first_rows, - std::vector* selected_row_groups) { - DORIS_CHECK(row_group_first_rows != nullptr); - DORIS_CHECK(selected_row_groups != nullptr); - row_group_first_rows->assign(metadata.num_row_groups(), 0); +} // namespace detail + +namespace detail { + +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups) { + DORIS_CHECK(row_group_first_rows != nullptr && selected_row_groups != nullptr); + if (scan_range.start_offset < 0 || scan_range.size < -1 || + (scan_range.size >= 0 && + scan_range.start_offset > std::numeric_limits::max() - scan_range.size)) { + return Status::Corruption("Invalid Parquet scan range [{}, {})", scan_range.start_offset, + scan_range.size); + } + const uint64_t range_start = static_cast(scan_range.start_offset); + const uint64_t range_end = scan_range.size < 0 + ? std::numeric_limits::max() + : range_start + static_cast(scan_range.size); + const size_t file_size = scan_range.file_size < 0 ? std::numeric_limits::max() + : static_cast(scan_range.file_size); + const bool full_file_range = + scan_range.size < 0 || (range_start == 0 && scan_range.file_size >= 0 && + range_end >= static_cast(scan_range.file_size)); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + row_group_first_rows->assign(metadata.row_groups.size(), 0); selected_row_groups->clear(); - selected_row_groups->reserve(metadata.num_row_groups()); - int64_t next_row_group_first_row = 0; - for (int row_group_idx = 0; row_group_idx < metadata.num_row_groups(); ++row_group_idx) { - (*row_group_first_rows)[row_group_idx] = next_row_group_first_row; - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows < 0) { + selected_row_groups->reserve(metadata.row_groups.size()); + int64_t next_first_row = 0; + for (size_t row_group_idx = 0; row_group_idx < metadata.row_groups.size(); ++row_group_idx) { + (*row_group_first_rows)[row_group_idx] = next_first_row; + const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { return Status::Corruption("Invalid negative row count in parquet row group {}", row_group_idx); } - next_row_group_first_row += row_group_rows; - if (!is_row_group_outside_range(metadata, scan_range, row_group_idx)) { - selected_row_groups->push_back(row_group_idx); + if (row_group.num_rows > std::numeric_limits::max() - next_first_row) { + return Status::Corruption("Parquet row counts overflow at row group {}", row_group_idx); + } + next_first_row += row_group.num_rows; + bool selected = full_file_range; + if (!full_file_range) { + if (row_group.columns.empty()) { + return Status::Corruption("Parquet row group {} has no column chunks", + row_group_idx); + } + size_t group_start = std::numeric_limits::max(); + size_t group_end = 0; + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, file_size, compat.parquet_816_padding, &chunk_range)); + group_start = std::min(group_start, chunk_range.offset); + group_end = std::max(group_end, chunk_range.offset + chunk_range.length); + } + // Checked chunk ranges make end >= start; this midpoint form cannot overflow even + // when footer offsets are close to the host coordinate limit. + const uint64_t group_mid = + static_cast(group_start) + (group_end - group_start) / 2; + selected = group_mid >= range_start && group_mid < range_end; + } + if (selected) { + selected_row_groups->push_back(cast_set(row_group_idx)); + } + } + return Status::OK(); +} + +} // namespace detail + +namespace { + +std::vector intersect_row_ranges(const std::vector& left, + const std::vector& right) { + std::vector result; + size_t left_idx = 0; + size_t right_idx = 0; + while (left_idx < left.size() && right_idx < right.size()) { + const int64_t left_end = left[left_idx].start + left[left_idx].length; + const int64_t right_end = right[right_idx].start + right[right_idx].length; + const int64_t start = std::max(left[left_idx].start, right[right_idx].start); + const int64_t end = std::min(left_end, right_end); + if (start < end) { + result.push_back({.start = start, .length = end - start}); + } + if (left_end < right_end) { + ++left_idx; + } else { + ++right_idx; + } + } + return result; +} + +Status finalize_native_row_group_read_plan( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, + RowGroupReadPlan* row_group_plan, ParquetPruningStats* pruning_stats, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + bool* selected) { + DORIS_CHECK(row_group_plan != nullptr && pruning_stats != nullptr && file_context != nullptr && + selected != nullptr); + *selected = true; + if (!row_group_plan->expensive_pruning_pending) { + return Status::OK(); + } + row_group_plan->expensive_pruning_pending = false; + const auto& thrift = metadata.to_thrift(); + const std::vector candidate {row_group_plan->row_group_id}; + std::vector metadata_selected; + RETURN_IF_ERROR(select_row_groups_by_metadata( + thrift, file_schema, request, &candidate, &metadata_selected, enable_bloom_filter, + pruning_stats, timezone, runtime_state, file_context, column_reader_profile, + ParquetMetadataProbeMode::EXPENSIVE_ONLY)); + if (metadata_selected.empty()) { + *selected = false; + return Status::OK(); + } + + std::unordered_set requested_leaf_ids; + for (const auto& projection : request_scan_columns(request)) { + const auto local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + continue; } + collect_projected_leaf_column_ids(*file_schema[local_id], projection, &requested_leaf_ids); + } + std::unordered_map page_indexes; + if (can_use_parquet_page_index(request, runtime_state)) { + RETURN_IF_ERROR(file_context->load_native_page_indexes( + row_group_plan->row_group_id, requested_leaf_ids, &page_indexes, + &pruning_stats->read_page_index_time, &pruning_stats->parse_page_index_time)); + } + std::vector page_selected_ranges; + std::map page_skip_plans; + RETURN_IF_ERROR(select_row_group_ranges_by_native_page_index( + thrift, page_indexes, file_schema, request, row_group_plan->row_group_rows, + &page_selected_ranges, &page_skip_plans, pruning_stats, timezone, runtime_state)); + row_group_plan->selected_ranges = + intersect_row_ranges(row_group_plan->selected_ranges, page_selected_ranges); + row_group_plan->page_skip_plans = std::move(page_skip_plans); + for (auto& [leaf_column_id, indexes] : page_indexes) { + row_group_plan->offset_indexes.emplace(leaf_column_id, std::move(indexes.offset_index)); + } + if (row_group_plan->selected_ranges.empty()) { + *selected = false; + return Status::OK(); } + pruning_stats->selected_row_ranges += row_group_plan->selected_ranges.size(); return Status::OK(); } -Status build_row_group_read_plans( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, +Status build_native_row_group_read_plans( + const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector& selected_row_groups, const std::vector& row_group_first_rows, RowGroupScanPlan* plan, - const cctz::time_zone* timezone, const RuntimeState* runtime_state) { - DORIS_CHECK(plan != nullptr); + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + const auto& thrift = metadata.to_thrift(); plan->row_groups.reserve(selected_row_groups.size()); - for (const auto row_group_idx : selected_row_groups) { - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(static_cast(row_group_idx) < row_group_first_rows.size()); - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows == 0) { + for (const int row_group_idx : selected_row_groups) { + const auto& row_group = thrift.row_groups[row_group_idx]; + if (row_group.num_rows == 0) { continue; } - RowGroupReadPlan row_group_plan; row_group_plan.row_group_id = row_group_idx; row_group_plan.first_file_row = row_group_first_rows[row_group_idx]; - row_group_plan.row_group_rows = row_group_rows; - RETURN_IF_ERROR(select_row_group_ranges_by_page_index( - file_reader, file_schema, request, row_group_idx, row_group_rows, - &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, - &plan->pruning_stats, timezone, runtime_state)); - if (row_group_plan.selected_ranges.empty()) { - continue; - } - plan->pruning_stats.selected_row_ranges += row_group_plan.selected_ranges.size(); + row_group_plan.row_group_rows = row_group.num_rows; + row_group_plan.selected_ranges = {{.start = 0, .length = row_group.num_rows}}; + row_group_plan.expensive_pruning_pending = true; plan->row_groups.push_back(std::move(row_group_plan)); } return Status::OK(); @@ -286,52 +511,58 @@ Status build_row_group_read_plans( } // namespace -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - DORIS_CHECK(plan != nullptr); + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); plan->row_groups.clear(); - plan->pruning_stats = ParquetPruningStats {}; - - // Row-group planning flow: - // - // parquet footer row groups - // | - // v - // split byte-range candidates - // | - // v - // row-group metadata pruning - // statistics/ZoneMap -> dictionary -> bloom filter - // | - // v - // page-index pruning per selected row group - // | - // v - // RowGroupReadPlan with selected row ranges - // - // Metadata pruning removes whole row groups before readers are opened. Page index pruning runs - // only for remaining row groups and produces selected row ranges; the scan scheduler later skips - // gaps between those ranges, while row-level VExpr conjuncts still run on loaded batches for - // correctness. + plan->pruning_stats = {}; + plan->enable_bloom_filter = enable_bloom_filter; std::vector row_group_first_rows; - std::vector scan_range_selected_row_groups; - RETURN_IF_ERROR(select_row_groups_by_scan_range(metadata, scan_range, &row_group_first_rows, - &scan_range_selected_row_groups)); - - std::vector metadata_selected_row_groups; + std::vector scan_range_selected; + RETURN_IF_ERROR(detail::select_native_row_groups_by_scan_range( + metadata.to_thrift(), scan_range, &row_group_first_rows, &scan_range_selected)); + std::vector metadata_selected; RETURN_IF_ERROR(select_row_groups_by_metadata( - metadata, file_reader, file_schema, request, &scan_range_selected_row_groups, - &metadata_selected_row_groups, enable_bloom_filter, &plan->pruning_stats, timezone, - runtime_state)); + metadata.to_thrift(), file_schema, request, &scan_range_selected, &metadata_selected, + enable_bloom_filter, &plan->pruning_stats, timezone, runtime_state, file_context, + column_reader_profile, ParquetMetadataProbeMode::FOOTER_ONLY)); + RETURN_IF_ERROR(build_native_row_group_read_plans(metadata, file_schema, request, + metadata_selected, row_group_first_rows, plan, + timezone, runtime_state, file_context)); + plan->pruning_stats.selected_row_groups = plan->row_groups.size(); + return Status::OK(); +} - RETURN_IF_ERROR(build_row_group_read_plans(metadata, file_reader, file_schema, request, - metadata_selected_row_groups, row_group_first_rows, - plan, timezone, runtime_state)); +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + std::vector selected_plans; + selected_plans.reserve(plan->row_groups.size()); + for (auto& row_group_plan : plan->row_groups) { + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + metadata, file_schema, request, enable_bloom_filter, &row_group_plan, + &deferred_stats, timezone, runtime_state, file_context, column_reader_profile, + &selected)); + if (parquet_profile != nullptr) { + parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (selected) { + selected_plans.push_back(std::move(row_group_plan)); + } + } + plan->row_groups = std::move(selected_plans); plan->pruning_stats.selected_row_groups = plan->row_groups.size(); return Status::OK(); } @@ -421,11 +652,12 @@ Status execute_compact_delete_conjuncts(const VExprContextSPtrs& delete_conjunct *can_filter_all = false; for (const auto& delete_conjunct : delete_conjuncts) { DORIS_CHECK(delete_conjunct != nullptr); + const size_t original_columns = file_block->columns(); int result_column_id = -1; RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, &result_column_id)); - DORIS_CHECK(result_column_id >= 0 && - result_column_id < static_cast(file_block->columns())); + RETURN_IF_ERROR(detail::validate_ephemeral_expr_result_column( + original_columns, result_column_id, file_block->columns())); const auto& delete_filter = assert_cast( *file_block->get_by_position(result_column_id).column) .get_data(); @@ -471,11 +703,12 @@ Status execute_delete_conjuncts(const format::FileScanRequest& request, int64_t break; } DORIS_CHECK(delete_conjunct != nullptr); + const size_t original_columns = file_block->columns(); int result_column_id = -1; RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, &result_column_id)); - DORIS_CHECK(result_column_id >= 0 && - result_column_id < static_cast(file_block->columns())); + RETURN_IF_ERROR(detail::validate_ephemeral_expr_result_column( + original_columns, result_column_id, file_block->columns())); const auto& delete_filter = assert_cast( *file_block->get_by_position(result_column_id).column) .get_data(); @@ -496,6 +729,19 @@ Status execute_delete_conjuncts(const format::FileScanRequest& request, int64_t } // namespace +Status detail::validate_ephemeral_expr_result_column(size_t original_columns, int result_column_id, + size_t current_columns) { + // Delete predicates may erase only a temporary expression result. A bare SlotRef returns an + // input column id, which must remain in the block for later predicates and materialization. + if (UNLIKELY(result_column_id < 0 || static_cast(result_column_id) < original_columns || + static_cast(result_column_id) >= current_columns)) { + return Status::InternalError( + "Delete conjunct result column {} is not ephemeral (original={}, current={})", + result_column_id, original_columns, current_columns); + } + return Status::OK(); +} + uint16_t apply_compact_filter_to_selection(const IColumn::Filter& filter, SelectionVector* selection, uint16_t selected_rows) { DORIS_CHECK(selection != nullptr); @@ -593,6 +839,7 @@ std::vector filter_ranges_by_condition_cache(const std::vectorflush_profile(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + reader->flush_profile(); + } +} + +bool ParquetScanScheduler::finish_current_reader_batch_profiles() { + bool crossed_page = false; + // A scheduler batch is counted once even when several projected leaves cross page boundaries. + for (const auto& reader : _current_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + return crossed_page; +} + +const detail::PredicateConjunctSchedule& ParquetScanScheduler::predicate_conjunct_schedule( + const format::FileScanRequest& request) { + if (_predicate_schedule_request == &request) { + return _predicate_schedule; + } + + // FileScanRequest is frozen by ParquetReader::open(). Its address therefore identifies both + // the conjunct set and local-position mapping for the scheduler lifetime. + _predicate_schedule = build_predicate_conjunct_schedule(request); + _predicate_schedule_request = &request; + _predicate_positions_scratch.clear(); + _predicate_indices_by_position_scratch.clear(); + _predicate_positions_scratch.reserve(request.predicate_columns.size()); + _predicate_indices_by_position_scratch.reserve(request.predicate_columns.size()); + for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + const auto position_it = + request.local_positions.find(request.predicate_columns[idx].column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + _predicate_positions_scratch.push_back(position); + _predicate_indices_by_position_scratch.emplace(position, idx); + } + return _predicate_schedule; +} + +std::vector ParquetScanScheduler::adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const { + std::vector positions; + std::unordered_map columns_by_position; + positions.reserve(request.predicate_columns.size()); + columns_by_position.reserve(request.predicate_columns.size()); + for (const auto& column : request.predicate_columns) { + const auto position_it = request.local_positions.find(column.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + positions.push_back(position); + columns_by_position.emplace(position, &column); + } + auto ordered = detail::order_adaptive_predicates(positions, _predicate_runtime_stats); + ordered = detail::adaptive_prefetch_prefix(ordered, _predicate_runtime_stats, 0.25); + std::vector result; + result.reserve(ordered.size()); + for (const size_t position : ordered) { + result.push_back(*columns_by_position.at(position)); + } + return result; +} + Status ParquetScanScheduler::open_next_row_group( ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, bool* has_row_group) { *has_row_group = false; - if (_next_row_group_plan_idx >= _row_group_plans.size()) { + RowGroupReadPlan* selected_plan = nullptr; + while (_next_row_group_plan_idx < _row_group_plans.size()) { + RowGroupReadPlan& candidate_plan = _row_group_plans[_next_row_group_plan_idx++]; + // Probe only the row group about to execute. This keeps LIMIT/cancellation latency + // independent of the number of later remote row groups while preserving eager footer + // statistics pruning during open. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + *file_context.native_metadata, file_schema, request, _enable_bloom_filter, + &candidate_plan, &deferred_stats, _timezone, _runtime_state, &file_context, + _scan_profile.column_reader_profile, &selected)); + if (_parquet_profile != nullptr) { + _parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (!selected) { + continue; + } + selected_plan = &candidate_plan; + break; + } + if (selected_plan == nullptr) { + // The last row group's native readers have already been released by + // reset_current_row_group(). Flush the shared merge reader now so its counters are visible + // when EOF is returned and its bounded scratch does not survive until file close. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; return Status::OK(); } - const RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; + RowGroupReadPlan& row_group_plan = *selected_plan; const int row_group_idx = row_group_plan.row_group_id; - // Row-level dictionary filters read dictionary pages before Arrow RecordReaders are created. - // Keep that probe on the base reader: MergeRangeFileReader expects each registered range to be - // consumed as one forward pass, while the later RecordReader opens the same column chunk again - // for the data-page stream. + // Dictionary probes and data-page readers share the native metadata tree. Reset the previous + // row-group merge reader before probing because dictionary-page offsets are not scan ordered. file_context.reset_random_access_ranges(); _current_merge_range_active = false; - try { - _current_row_group = file_context.file_reader->RowGroup(row_group_idx); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } - - auto row_group_metadata = file_context.metadata->RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - _current_row_group_rows = row_group_metadata->num_rows(); + + const auto& row_group_metadata = + file_context.native_metadata->to_thrift().row_groups[row_group_idx]; + _current_row_group_rows = row_group_metadata.num_rows; DORIS_CHECK(_current_row_group_rows == row_group_plan.row_group_rows); DORIS_CHECK(_current_row_group_rows > 0); _current_row_group_id = row_group_idx; + _has_current_row_group = true; DORIS_CHECK(!row_group_plan.selected_ranges.empty()); _current_row_group_first_row = row_group_plan.first_file_row; _current_row_group_rows_read = 0; _current_selected_ranges = row_group_plan.selected_ranges; + _current_offset_indexes = std::move(row_group_plan.offset_indexes); + // Condition Cache and split planning can narrow logical ranges without a physical OffsetIndex. + // Native readers must keep the sequential level/value cursor path valid in that case; only a + // PageIndex-derived skip plan requires the transferred indexes below. + for (const auto& [leaf_column_id, skip_plan] : row_group_plan.page_skip_plans) { + if (!_current_offset_indexes.contains(leaf_column_id)) { + continue; + } + for (size_t page = 0; page < skip_plan.skipped_pages.size(); ++page) { + if (!skip_plan.should_skip_page(page)) { + continue; + } + if (_page_skip_profile.skipped_pages != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); + } + if (_page_skip_profile.skipped_bytes != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_bytes, + skip_plan.skipped_page_compressed_size(page)); + } + } + } _current_range_idx = 0; _current_range_rows_read = 0; _current_predicate_columns.clear(); _current_non_predicate_columns.clear(); _current_dictionary_filters.clear(); RETURN_IF_ERROR(prepare_current_dictionary_filters(file_context, file_schema, request, - row_group_idx, *row_group_metadata)); - _current_merge_range_active = - prepare_current_row_group_reader(file_context, file_schema, request, row_group_idx); - - ParquetColumnReaderFactory column_reader_factory( - _current_row_group, file_context.schema->num_columns(), &row_group_plan.page_skip_plans, - _page_skip_profile, _timezone, _enable_strict_mode, - _scan_profile.column_reader_profile); + row_group_idx, row_group_metadata)); + // Dictionary probing is complete, so the native data-page readers can now share the same + // row-group-scoped MergeRangeFileReader policy as v1. Sharing one wrapper is important: a + // separate merge reader per leaf would duplicate its 128MB scratch capacity and defeat lazy + // materialization for wide schemas. + const auto& thrift_metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : std::string {}); + std::vector native_ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + thrift_metadata, file_schema, request_scan_columns(request), row_group_idx, + file_context.native_file->size(), compat.parquet_816_padding, &native_ranges)); + _current_merge_range_active = file_context.set_native_random_access_ranges( + native_ranges, detail::average_prefetch_range_size(native_ranges), _profile, + _merge_read_slice_size); + for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + const auto local_id = col.column_id(); + if (_current_predicate_columns.contains(local_id)) { + continue; + } + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { + _current_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + _current_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR( - column_reader_factory.create(*column_schema, &col, &column_reader, - _current_dictionary_filters.contains(local_id))); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, + _current_dictionary_filters.contains(local_id), _scan_profile.column_reader_profile, + &column_reader)); _current_predicate_columns[local_id] = std::move(column_reader); } - // Start warming filter-column chunks as soon as their row group is selected. Parquet v2 still - // reads through Arrow's random-access reader; this prefetch only warms Doris file cache blocks - // in the background and never changes the row/column materialization order. + // Start warming filter-column chunks as soon as their row group is selected. The native + // BufferedFileStreamReader later consumes the same Doris file-cache blocks; prefetch never + // changes row/column materialization order. if (!_current_merge_range_active) { - prefetch_current_row_group_columns(file_context, file_schema, request.predicate_columns, - &_current_predicate_prefetched); + const auto prefetch_columns = adaptive_predicate_prefetch_columns(request); + RETURN_IF_ERROR(prefetch_current_row_group_columns( + file_context, file_schema, prefetch_columns, &_current_predicate_prefetched)); } for (const auto& col : request.non_predicate_columns) { - const auto local_id = col.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_non_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + const auto local_id = col.column_id(); + if (request.is_count_star_placeholder(col.column_id())) { + continue; + } + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { + _current_non_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_non_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + _current_non_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR(column_reader_factory.create(*column_schema, &col, &column_reader)); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, false, _scan_profile.column_reader_profile, + &column_reader)); _current_non_predicate_columns[local_id] = std::move(column_reader); } - if (!_current_merge_range_active && request.conjuncts.empty() && - request.delete_conjuncts.empty()) { + if (!_current_merge_range_active && + ((request.conjuncts.empty() && request.delete_conjuncts.empty()) || + _predicate_survival_ratio >= 0.8)) { // With no row-level filters there is no lazy-read decision to wait for, so start warming // output chunks immediately after their readers are created. Filtered scans still defer // this until at least one row survives the predicate phase. - prefetch_current_row_group_columns(file_context, file_schema, request.non_predicate_columns, - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); } *has_row_group = true; return Status::OK(); @@ -787,21 +1192,29 @@ Status ParquetScanScheduler::skip_current_row_group_rows(int64_t rows) { for (const auto& column_reader : _current_predicate_columns | std::views::values) { RETURN_IF_ERROR(column_reader->skip(rows)); } + // Keep page-index/condition-cache gaps pending for lazy columns as well. For example, after a + // fully filtered [0, 32) batch and a pruned [32, 96) gap, predicate readers are at 96 while lazy + // readers remain at 0; one later skip(96) is cheaper than skip(32) followed by skip(64). + DORIS_CHECK(_pending_non_predicate_skip_rows <= std::numeric_limits::max() - rows); + _pending_non_predicate_skip_rows += rows; + _current_row_group_rows_read += rows; + return Status::OK(); +} + +Status ParquetScanScheduler::flush_pending_non_predicate_skip_rows() { + if (_pending_non_predicate_skip_rows == 0) { + return Status::OK(); + } for (const auto& column_reader : _current_non_predicate_columns | std::views::values) { - RETURN_IF_ERROR(column_reader->skip(rows)); + RETURN_IF_ERROR(column_reader->skip(_pending_non_predicate_skip_rows)); } - _current_row_group_rows_read += rows; + _pending_non_predicate_skip_rows = 0; return Status::OK(); } namespace { -struct PredicateConjunctSchedule { - std::map single_column_conjuncts; - VExprContextSPtrs remaining_conjuncts; -}; - -PredicateConjunctSchedule build_predicate_conjunct_schedule( +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( const format::FileScanRequest& request) { std::unordered_set predicate_block_positions; predicate_block_positions.reserve(request.predicate_columns.size()); @@ -811,7 +1224,7 @@ PredicateConjunctSchedule build_predicate_conjunct_schedule( predicate_block_positions.insert(position_it->second.value()); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; for (const auto& conjunct : request.conjuncts) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); @@ -909,21 +1322,11 @@ uint16_t count_selected_rows(const IColumn::Filter& filter) { return selected_rows; } -Status filter_read_predicate_columns(Block* file_block, const std::vector& positions, - const IColumn::Filter& compact_filter) { - if (positions.empty()) { - return Status::OK(); - } - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(file_block, positions, compact_filter)); - return Status::OK(); -} - IColumn::Filter build_dictionary_entry_filter(size_t block_position, const ParquetColumnSchema& column_schema, const VExprContextSPtrs& conjuncts, - const ParquetDictionaryWords& dict_words) { - auto fields = dictionary_fields_from_words(dict_words); - IColumn::Filter dictionary_filter(fields.size(), 1); + const IColumn& dictionary) { + IColumn::Filter dictionary_filter(dictionary.size(), 1); DictionaryEvalContext ctx; auto& slot = ctx.slots .emplace(static_cast(block_position), @@ -931,14 +1334,15 @@ IColumn::Filter build_dictionary_entry_filter(size_t block_position, .data_type = column_schema.type, .values = {}}) .first->second; slot.values.reserve(1); - - for (size_t dict_idx = 0; dict_idx < fields.size(); ++dict_idx) { + for (size_t dictionary_id = 0; dictionary_id < dictionary.size(); ++dictionary_id) { + Field value; + dictionary.get(dictionary_id, value); slot.values.clear(); - slot.values.push_back(fields[dict_idx]); - dictionary_filter[dict_idx] = VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch - ? 0 - : 1; + slot.values.push_back(std::move(value)); + dictionary_filter[dictionary_id] = VExprContext::evaluate_dictionary_filter( + conjuncts, ctx) == ZoneMapFilterResult::kNoMatch + ? 0 + : 1; } return dictionary_filter; } @@ -949,16 +1353,16 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata) { + const tparquet::RowGroup& row_group_metadata) { _current_dictionary_filters.clear(); _current_dictionary_residual_conjuncts.clear(); if (request.conjuncts.empty()) { return Status::OK(); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; { SCOPED_TIMER(_scan_profile.dict_filter_expr_rewrite_time); - schedule = build_predicate_conjunct_schedule(request); + schedule = predicate_conjunct_schedule(request); } if (schedule.single_column_conjuncts.empty()) { return Status::OK(); @@ -966,8 +1370,8 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( SCOPED_TIMER(_scan_profile.dict_filter_rewrite_time); for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); - if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + const auto local_id = col.column_id(); + if (!local_id.is_valid() || local_id.value() >= static_cast(file_schema.size())) { continue; } const auto position_it = request.local_positions.find(col.column_id()); @@ -983,29 +1387,38 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( // This optimization is deliberately limited to single-column predicates with a dictionary // evaluable part. Mixed AND predicates are split so dictionary-covered children run as a // dict-id prefilter and residual children keep the normal row-level expression path. - const auto& column_schema = file_schema[local_id]; + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); if (column_schema->leaf_column_id < 0 || - column_schema->leaf_column_id >= row_group_metadata.num_columns()) { + column_schema->leaf_column_id >= static_cast(row_group_metadata.columns.size())) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } - auto column_chunk = row_group_metadata.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_row_level_dictionary_filter(*column_schema, *column_chunk)) { + const auto& column_chunk = row_group_metadata.columns[column_schema->leaf_column_id]; + if (!column_chunk.__isset.meta_data || + !supports_row_level_dictionary_filter(*column_schema, column_chunk.meta_data)) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } - ParquetDictionaryWords dict_words; + std::unique_ptr column_reader; + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_file, file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, true, _scan_profile.column_reader_profile, + &column_reader)); + MutableColumnPtr dictionary_values; { SCOPED_TIMER(_scan_profile.dict_filter_read_dict_time); - if (!read_dictionary_words(file_context.file_reader.get(), row_group_idx, - column_schema->leaf_column_id, *column_schema, - &dict_words)) { + auto dictionary_result = column_reader->dictionary_values(); + if (!dictionary_result.has_value()) { update_counter_if_not_null(_scan_profile.dict_filter_read_failures, 1); + // Dictionary filtering is optional: a probe failure must not reject a file that + // the normal native read path can still decode. continue; } + dictionary_values = std::move(dictionary_result).value(); } // Build a safe dictionary prefilter from the dictionary-filter interface instead of @@ -1016,15 +1429,16 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( DictionaryResidualConjuncts residual_conjuncts; { SCOPED_TIMER(_scan_profile.dict_filter_build_time); - dictionary_filter = build_dictionary_entry_filter(block_position, *column_schema, - conjunct_it->second, dict_words); + dictionary_filter = build_dictionary_entry_filter( + block_position, *column_schema, conjunct_it->second, *dictionary_values); residual_conjuncts = build_dictionary_residual_conjuncts(conjunct_it->second); } // The bitmap is keyed by Parquet dictionary id. Later data-page reads evaluate the - // predicate with an integer lookup and only materialize STRING values for surviving rows. + // predicate with an integer lookup and materialize typed values only for surviving rows. _current_dictionary_filters.emplace(local_id, std::move(dictionary_filter)); _current_dictionary_residual_conjuncts.emplace(local_id, std::move(residual_conjuncts)); + _current_predicate_columns.emplace(local_id, std::move(column_reader)); update_counter_if_not_null(_scan_profile.dict_filter_columns, 1); } return Status::OK(); @@ -1041,16 +1455,122 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, if (!request.conjuncts.empty() || !request.delete_conjuncts.empty()) { selection->resize(static_cast(batch_rows)); } - const auto schedule = build_predicate_conjunct_schedule(request); + const auto& schedule = predicate_conjunct_schedule(request); + std::unordered_set residual_predicate_positions; + auto remember_residual_positions = [&](const VExprContextSPtrs& conjuncts) { + for (const auto& conjunct : conjuncts) { + std::set positions; + conjunct->root()->collect_slot_column_ids(positions); + for (const int position : positions) { + if (position >= 0) { + residual_predicate_positions.insert(cast_set(position)); + } + } + } + }; + remember_residual_positions(schedule.remaining_conjuncts); + remember_residual_positions(request.delete_conjuncts); + const size_t predicate_batch_sequence = _predicate_batch_sequence++; const bool can_read_predicate_columns_round_by_round = !schedule.single_column_conjuncts.empty(); - std::vector read_column_positions; + auto& read_column_positions = _read_column_positions_scratch; + read_column_positions.clear(); read_column_positions.reserve(request.predicate_columns.size()); + for (auto& rows : _predicate_column_selection_scratch | std::views::values) { + rows.clear(); + } + + auto remember_column_selection = [&](uint32_t position) { + auto& rows = _predicate_column_selection_scratch[position]; + rows.resize(*selected_rows); + for (uint16_t row = 0; row < *selected_rows; ++row) { + // SelectionVector and the scanner batch contract both bound row ordinals to uint16_t; + // keep the checked conversion explicit when persisting the coordinate mapping. + rows[row] = cast_set(selection->get_index(row)); + } + }; + + auto compact_predicate_columns = [&](bool discard_predicate_only_payload) -> Status { + bool compacted = false; + int64_t compacted_bytes = 0; + for (const uint32_t position : read_column_positions) { + auto& source_rows = _predicate_column_selection_scratch[position]; + const auto& old_column = file_block->get_by_position(position).column; + if (old_column->size() != source_rows.size()) { + return Status::Corruption( + "Predicate column {} has {} values but {} remembered source rows", position, + old_column->size(), source_rows.size()); + } + bool predicate_only = false; + if (discard_predicate_only_payload) { + predicate_only = std::ranges::any_of( + request.predicate_only_columns, [&](format::LocalColumnId local_id) { + const auto position_it = request.local_positions.find(local_id); + return position_it != request.local_positions.end() && + position_it->second.value() == position; + }); + } + if (predicate_only) { + auto placeholder = old_column->clone_empty(); + // Hidden predicate values are dead after the last filter, but every file-block + // column must retain the selected row count until TableReader drops hidden slots. + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(position, std::move(placeholder)); + remember_column_selection(position); + continue; + } + bool already_compact = source_rows.size() == *selected_rows && + old_column->size() == static_cast(*selected_rows); + for (uint16_t row = 0; already_compact && row < *selected_rows; ++row) { + already_compact = source_rows[row] == selection->get_index(row); + } + if (already_compact) { + continue; + } + auto& filter = _predicate_compaction_filter_scratch; + // resize_fill() preserves bytes when the next predicate column is smaller. Clear the + // whole reusable mask so survivors from an earlier coordinate space cannot reappear. + filter.resize(source_rows.size()); + std::ranges::fill(filter, 0); + size_t source_idx = 0; + uint16_t selected_idx = 0; + while (source_idx < source_rows.size() && selected_idx < *selected_rows) { + const auto source_row = source_rows[source_idx]; + const auto selected_row = selection->get_index(selected_idx); + if (source_row < selected_row) { + ++source_idx; + continue; + } + DORIS_CHECK_EQ(source_row, selected_row); + filter[source_idx++] = 1; + ++selected_idx; + } + DORIS_CHECK_EQ(selected_idx, *selected_rows); + compacted_bytes += static_cast(old_column->byte_size()); + RETURN_IF_CATCH_EXCEPTION(file_block->replace_by_position( + position, old_column->filter(filter, *selected_rows))); + remember_column_selection(position); + compacted = true; + } + if (compacted) { + update_counter_if_not_null(_scan_profile.predicate_compaction_bytes, compacted_bytes); + update_counter_if_not_null(_scan_profile.predicate_compaction_count, 1); + } + // The output path must not apply a batch-coordinate filter to columns that now use compact + // coordinates. The loop above establishes this invariant even when no bytes moved because + // every column was already aligned. + *predicate_columns_filtered = !read_column_positions.empty(); + return Status::OK(); + }; - auto read_predicate_column = [&](ParquetColumnReader* column_reader, size_t block_position, - ColumnId local_id, bool* used_dictionary_filter) -> Status { + auto read_predicate_column = + [&](ParquetColumnReader* column_reader, size_t block_position, + format::LocalColumnId local_id, const VExprContextSPtrs* single_column_conjuncts, + bool* used_dictionary_filter, bool* used_fixed_width_filter) -> Status { DORIS_CHECK(used_dictionary_filter != nullptr); + DORIS_CHECK(used_fixed_width_filter != nullptr); *used_dictionary_filter = false; + *used_fixed_width_filter = false; DCHECK(remove_nullable(column_reader->type()) ->equals(*remove_nullable(file_block->get_by_position(block_position).type))) << column_reader->type()->get_name() << " " @@ -1077,22 +1597,72 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, update_counter_if_not_null(_scan_profile.rows_filtered_by_dict_filter, filtered_rows); if (new_selected_rows != selected_rows_before) { - // The dictionary reader has already appended only surviving values for the - // current column. Apply the compact row filter only to columns read before this - // one, then update the shared selection for later predicate/lazy columns. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); + // The dictionary reader already appended only survivors for this column. Keep + // older predicate columns in their original coordinate spaces and compact all + // of them once at the expression/output boundary below. *selected_rows = apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); *used_dictionary_filter = true; return Status::OK(); } } + if (single_column_conjuncts != nullptr && + !residual_predicate_positions.contains(block_position)) { + VExprSPtrs direct_conjuncts; + direct_conjuncts.reserve(single_column_conjuncts->size()); + std::ranges::transform(*single_column_conjuncts, std::back_inserter(direct_conjuncts), + [](const auto& context) { return context->root(); }); + if (!direct_conjuncts.empty()) { + const uint16_t selected_rows_before = *selected_rows; + IColumn::Filter compact_filter; + bool used_filter = false; + const bool predicate_only = request.is_predicate_only(local_id); + // The raw decoder cannot rewind after evaluating encoded fixed-width values. + // Project survivors in that pass when output still needs the predicate column. + IColumn* projected_column = predicate_only ? nullptr : column.get(); + RETURN_IF_ERROR(column_reader->select_with_fixed_width_filter( + *selection, *selected_rows, batch_rows, direct_conjuncts, + cast_set(block_position), projected_column, &compact_filter, + &used_filter)); + if (used_filter) { + DORIS_CHECK_EQ(compact_filter.size(), selected_rows_before); + update_counter_if_not_null(_scan_profile.fixed_width_predicate_direct_batches, + 1); + update_counter_if_not_null(_scan_profile.fixed_width_predicate_direct_rows, + selected_rows_before); + const uint16_t new_selected_rows = count_selected_rows(compact_filter); + const auto filtered_rows = static_cast(selected_rows_before) - + static_cast(new_selected_rows); + if (conjunct_filtered_rows != nullptr) { + *conjunct_filtered_rows += filtered_rows; + } + if (new_selected_rows != selected_rows_before) { + *selected_rows = apply_compact_filter_to_selection( + compact_filter, selection, selected_rows_before); + } + if (predicate_only) { + // This slot is absent from every residual/delete conjunct, so no later + // expression can observe its payload. Keep only the block row-shape contract. + auto placeholder = column->clone_empty(); + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(block_position, std::move(placeholder)); + } else { + file_block->replace_by_position(block_position, std::move(column)); + } + read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); + *predicate_columns_filtered = true; + *used_fixed_width_filter = true; + return Status::OK(); + } + } + } + if (*selected_rows == batch_rows) { int64_t column_rows = 0; RETURN_IF_ERROR(column_reader->read(batch_rows, column, &column_rows)); @@ -1113,6 +1683,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); return Status::OK(); }; @@ -1134,16 +1705,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // All columns read so far are already compacted to the current selection. Apply the - // compact filter to those columns and the selection vector together, so later predicate - // columns can read only rows that survived previous predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1167,16 +1732,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // Dictionary-covered children have already reduced the compact block. Apply only the - // residual child filters here, then keep the same compacted-column invariant as the - // normal conjunct path for later predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1213,24 +1772,23 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, compact_filter.resize_fill(selected_rows_before, 0); } if (can_filter_all || count_selected_rows(compact_filter) != selected_rows_before) { - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; auto read_all_predicate_columns = [&]() -> Status { for (const auto& [fid, column_reader] : _current_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); bool used_dictionary_filter = false; + bool used_fixed_width_filter = false; RETURN_IF_ERROR(read_predicate_column(column_reader.get(), position_it->second.value(), - fid, &used_dictionary_filter)); + fid, nullptr, &used_dictionary_filter, + &used_fixed_width_filter)); } return Status::OK(); }; @@ -1250,45 +1808,69 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, // Single-column conjuncts can be evaluated immediately after their column is read. Once // selection shrinks, later predicate columns use ParquetColumnReader::select() so the // reader skips rows already rejected by earlier predicates instead of materializing them. - for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + _ordered_predicate_positions_scratch = detail::order_adaptive_predicates( + _predicate_positions_scratch, _predicate_runtime_stats); + const auto& ordered_positions = _ordered_predicate_positions_scratch; + for (size_t order_idx = 0; order_idx < ordered_positions.size(); ++order_idx) { + const size_t position = ordered_positions[order_idx]; + const size_t idx = _predicate_indices_by_position_scratch.at(position); const auto& col = request.predicate_columns[idx]; - const auto fid = col.local_id(); + const auto fid = col.column_id(); auto reader_it = _current_predicate_columns.find(fid); DORIS_CHECK(reader_it != _current_predicate_columns.end()); auto position_it = request.local_positions.find(col.column_id()); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); + const uint16_t rows_before = *selected_rows; + auto& stats = _predicate_runtime_stats[position]; + const bool sample = detail::should_sample_adaptive_predicate(stats.samples, + predicate_batch_sequence); + const int64_t start_ns = sample ? MonotonicNanos() : 0; bool used_dictionary_filter = false; + bool used_fixed_width_filter = false; + const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); + const VExprContextSPtrs* column_conjuncts = + conjunct_it == schedule.single_column_conjuncts.end() ? nullptr + : &conjunct_it->second; RETURN_IF_ERROR(read_predicate_column(reader_it->second.get(), block_position, fid, - &used_dictionary_filter)); - if (*selected_rows == 0) { - for (size_t remaining_idx = idx + 1; - remaining_idx < request.predicate_columns.size(); ++remaining_idx) { - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); - auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); - DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); - RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); + column_conjuncts, &used_dictionary_filter, + &used_fixed_width_filter)); + if (*selected_rows != 0 && conjunct_it != schedule.single_column_conjuncts.end()) { + if (used_dictionary_filter) { + const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); + DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); + RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( + residual_it->second)); + } else if (!used_fixed_width_filter) { + RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); } - return Status::OK(); - } - const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); - if (conjunct_it == schedule.single_column_conjuncts.end()) { - continue; } - if (used_dictionary_filter) { - const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); - DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); - RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( - residual_it->second)); - } else { - RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); + if (sample) { + const double cost_per_row = static_cast(MonotonicNanos() - start_ns) / + std::max(rows_before, 1); + const double survival = + static_cast(*selected_rows) / std::max(rows_before, 1); + constexpr double ADAPTIVE_ALPHA = 0.25; + if (stats.samples == 0) { + stats.cost_per_input_row_ns = cost_per_row; + stats.survival_ratio = survival; + } else { + stats.cost_per_input_row_ns = + ADAPTIVE_ALPHA * cost_per_row + + (1 - ADAPTIVE_ALPHA) * stats.cost_per_input_row_ns; + stats.survival_ratio = + ADAPTIVE_ALPHA * survival + (1 - ADAPTIVE_ALPHA) * stats.survival_ratio; + } + ++stats.samples; } if (*selected_rows != 0) { continue; } - for (size_t remaining_idx = idx + 1; remaining_idx < request.predicate_columns.size(); - ++remaining_idx) { - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); + for (size_t remaining_order_idx = order_idx + 1; + remaining_order_idx < ordered_positions.size(); ++remaining_order_idx) { + const size_t remaining_idx = _predicate_indices_by_position_scratch.at( + ordered_positions[remaining_order_idx]); + const auto remaining_fid = request.predicate_columns[remaining_idx].column_id(); auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); @@ -1298,47 +1880,59 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, return Status::OK(); }; + auto compact_predicate_columns_with_profile = + [&](bool discard_predicate_only_payload) -> Status { + const int64_t start_ns = MonotonicNanos(); + auto status = compact_predicate_columns(discard_predicate_only_payload); + update_counter_if_not_null(_scan_profile.predicate_compaction_time, + MonotonicNanos() - start_ns); + return status; + }; + RETURN_IF_ERROR(read_round_by_round()); + // Single-column expressions only touch the just-read column, so earlier columns can retain + // their own row mappings. Compact only when a later expression needs a shared coordinate + // space; otherwise the final boundary can discard hidden predicate payloads without scanning + // them again. + if (!schedule.remaining_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); + } RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(schedule.remaining_conjuncts)); - if (_scan_profile.predicate_filter_time == nullptr) { - return execute_scheduled_delete_conjuncts(); + if (!request.delete_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); } - SCOPED_TIMER(_scan_profile.predicate_filter_time); - return execute_scheduled_delete_conjuncts(); -} - -bool ParquetScanScheduler::prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx) { - if (file_context.metadata == nullptr) { - return false; + if (_scan_profile.predicate_filter_time == nullptr) { + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); + } else { + SCOPED_TIMER(_scan_profile.predicate_filter_time); + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); } - const auto ranges = build_row_group_prefetch_ranges( - *file_context.metadata, file_schema, request_scan_columns(request), row_group_idx); - const size_t avg_io_size = detail::average_prefetch_range_size(ranges); - return file_context.set_random_access_ranges(ranges, avg_io_size, _profile, - _merge_read_slice_size); + return compact_predicate_columns_with_profile(true); } -void ParquetScanScheduler::prefetch_current_row_group_columns( +Status ParquetScanScheduler::prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched) { DORIS_CHECK(prefetched != nullptr); if (_current_merge_range_active || *prefetched || scan_columns.empty() || - _current_row_group_id < 0 || file_context.metadata == nullptr) { - return; + _current_row_group_id < 0 || file_context.native_metadata == nullptr) { + return Status::OK(); } *prefetched = true; // The scanner request separates predicate and non-predicate columns so Parquet can read // predicate columns first and lazily materialize the rest. Keep the same contract for // prefetch: callers decide which side to warm, and this helper only translates that selected // projection into physical column-chunk byte ranges for the current row group. - file_context.prefetch_ranges( - build_row_group_prefetch_ranges(*file_context.metadata, file_schema, scan_columns, - _current_row_group_id), - nullptr); + const auto& metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + std::vector ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + metadata, file_schema, scan_columns, _current_row_group_id, + file_context.native_file->size(), compat.parquet_816_padding, &ranges)); + file_context.prefetch_ranges(ranges, nullptr); + return Status::OK(); } Status ParquetScanScheduler::read_current_row_group_batch( @@ -1346,6 +1940,24 @@ Status ParquetScanScheduler::read_current_row_group_batch( const std::vector>& file_schema, int64_t batch_rows, const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, size_t* rows) { + // Reader statistics are cumulative plain integers. Publishing their delta recursively for + // every tiny batch is measurable on wide/nested scans, so flush periodically and force the + // tail at row-group reset/close. + Defer profile_flush {[this, batch_rows]() { + // A widened predicate batch can be emitted in several output slices. Its lazy readers + // have not consumed the whole physical batch until the last slice is drained. + if (_pending_predicate_selection.empty() && finish_current_reader_batch_profiles() && + _scan_profile.column_reader_profile.page_crossing_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.column_reader_profile.page_crossing_batches, 1); + } + const bool finishes_row_group = _current_range_idx + 1 == _current_selected_ranges.size() && + _current_range_rows_read + batch_rows == + _current_selected_ranges[_current_range_idx].length; + if (++_batches_since_profile_flush >= PROFILE_FLUSH_BATCH_INTERVAL || finishes_row_group) { + flush_current_reader_profiles(); + _batches_since_profile_flush = 0; + } + }}; if (_scan_profile.total_batches != nullptr) { COUNTER_UPDATE(_scan_profile.total_batches, 1); } @@ -1355,12 +1967,13 @@ Status ParquetScanScheduler::read_current_row_group_batch( _raw_rows_read += batch_rows; if (_current_predicate_columns.empty() && _current_non_predicate_columns.empty()) { *rows = static_cast(batch_rows); + materialize_count_star_placeholders(request, *rows, file_block); if (_scan_profile.selected_rows != nullptr) { COUNTER_UPDATE(_scan_profile.selected_rows, batch_rows); } return Status::OK(); } - SelectionVector selection; + auto& selection = _selection; DORIS_CHECK(batch_rows <= std::numeric_limits::max()); uint16_t selected_rows = static_cast(batch_rows); int64_t conjunct_filtered_rows = 0; @@ -1371,6 +1984,10 @@ Status ParquetScanScheduler::read_current_row_group_batch( mark_condition_cache_granules(selection, selected_rows, batch_first_file_row); const bool need_filter_output = selected_rows != batch_rows; + const double batch_survival = static_cast(selected_rows) / batch_rows; + _predicate_survival_ratio = _predicate_survival_ratio < 0 + ? batch_survival + : 0.25 * batch_survival + 0.75 * _predicate_survival_ratio; if (_scan_profile.selected_rows != nullptr) { COUNTER_UPDATE(_scan_profile.selected_rows, selected_rows); } @@ -1383,6 +2000,11 @@ Status ParquetScanScheduler::read_current_row_group_batch( } if (selected_rows == 0 && _scan_profile.empty_selection_batches != nullptr) { COUNTER_UPDATE(_scan_profile.empty_selection_batches, 1); + } else if (static_cast(selected_rows) == batch_rows && + _scan_profile.dense_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.dense_batches, 1); + } else if (_scan_profile.selected_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.selected_batches, 1); } if (need_filter_output && !predicate_columns_filtered) { IColumn::Filter output_filter = selection_to_filter(selection, selected_rows, batch_rows); @@ -1395,19 +2017,56 @@ Status ParquetScanScheduler::read_current_row_group_batch( .column->filter(output_filter, selected_rows))); } } + if (selected_rows == 0) { + // Predicate readers have consumed this physical batch, but touching every lazy column here + // turns a long rejected prefix into `empty_batches * lazy_columns` native calls. Record only + // the positional lag. If [0, 32), [32, 64), and [64, 96) are empty, the first surviving + // batch performs one skip(96) per lazy column. If the row group ends instead, reset drops the + // lazy readers without flushing because no value from them can be observed. + DORIS_CHECK(_pending_non_predicate_skip_rows <= + std::numeric_limits::max() - batch_rows); + _pending_non_predicate_skip_rows += batch_rows; + *rows = 0; + return Status::OK(); + } if (!_current_merge_range_active && selected_rows > 0 && !_current_non_predicate_columns.empty()) { // Do not prefetch lazy output columns until at least one row survives filtering. This is // the same decision point where the v2 reader switches from predicate-only reads to // materializing non-predicate columns, so fully filtered batches avoid unnecessary IO. - prefetch_current_row_group_columns(file_context, file_schema, request.non_predicate_columns, - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); + } + + if (selected_rows > _batch_size) { + DORIS_CHECK(_pending_predicate_selection.empty()); + _pending_predicate_batch_rows = batch_rows; + _pending_predicate_batch_rows_consumed = 0; + _pending_predicate_selected_offset = 0; + _pending_predicate_selection.resize(selected_rows); + for (uint16_t idx = 0; idx < selected_rows; ++idx) { + _pending_predicate_selection[idx] = + static_cast(selection.get_index(idx)); + } + for (const auto& col : request.predicate_columns) { + const auto position_it = request.local_positions.find(col.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t block_position = position_it->second.value(); + const auto& column = file_block->get_by_position(block_position).column; + DORIS_CHECK_EQ(column->size(), selected_rows); + _pending_predicate_columns.emplace(block_position, column); + } + return materialize_pending_predicate_batch(request, file_block, rows); } { SCOPED_TIMER(_scan_profile.column_read_time); + // Bring lazy readers to the first row of the current physical batch before interpreting its + // selection vector. This also merges pending range gaps with fully filtered batches. + RETURN_IF_ERROR(flush_pending_non_predicate_skip_rows()); for (const auto& [fid, column_reader] : _current_non_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); auto column = file_block->get_by_position(block_position).column->assert_mutable(); @@ -1438,10 +2097,81 @@ Status ParquetScanScheduler::read_current_row_group_batch( file_block->replace_by_position(block_position, std::move(column)); } } + materialize_count_star_placeholders(request, selected_rows, file_block); *rows = static_cast(selected_rows); return Status::OK(); } +Status ParquetScanScheduler::materialize_pending_predicate_batch( + const format::FileScanRequest& request, Block* file_block, size_t* rows) { + DORIS_CHECK(!_pending_predicate_selection.empty()); + DORIS_CHECK(_pending_predicate_selected_offset < _pending_predicate_selection.size()); + const size_t remaining_selected = + _pending_predicate_selection.size() - _pending_predicate_selected_offset; + const size_t output_rows = + std::min(static_cast(_batch_size), remaining_selected); + const size_t output_end = _pending_predicate_selected_offset + output_rows; + const int64_t physical_end = + output_end == _pending_predicate_selection.size() + ? _pending_predicate_batch_rows + : static_cast(_pending_predicate_selection[output_end - 1]) + 1; + DORIS_CHECK(physical_end > _pending_predicate_batch_rows_consumed); + const int64_t physical_rows = physical_end - _pending_predicate_batch_rows_consumed; + + _pending_output_selection.resize(output_rows); + for (size_t idx = 0; idx < output_rows; ++idx) { + const int64_t physical_row = + _pending_predicate_selection[_pending_predicate_selected_offset + idx]; + DORIS_CHECK(physical_row >= _pending_predicate_batch_rows_consumed); + _pending_output_selection.set_index( + idx, static_cast(physical_row - + _pending_predicate_batch_rows_consumed)); + } + + for (const auto& [block_position, column] : _pending_predicate_columns) { + file_block->replace_by_position( + block_position, column->cut(_pending_predicate_selected_offset, output_rows)); + } + { + SCOPED_TIMER(_scan_profile.column_read_time); + RETURN_IF_ERROR(flush_pending_non_predicate_skip_rows()); + for (const auto& [fid, column_reader] : _current_non_predicate_columns) { + auto position_it = request.local_positions.find(fid); + DORIS_CHECK(position_it != request.local_positions.end()); + const auto block_position = position_it->second.value(); + auto column = file_block->get_by_position(block_position).column->assert_mutable(); + [[maybe_unused]] const auto old_size = column->size(); + RETURN_IF_ERROR(column_reader->select(_pending_output_selection, + static_cast(output_rows), physical_rows, + column)); + if (column->size() != old_size + output_rows) { + return Status::Corruption( + "Parquet pending output column {} returned {} rows, expected {} rows", + column_reader->name(), column->size(), old_size + output_rows); + } + file_block->replace_by_position(block_position, std::move(column)); + } + } + materialize_count_star_placeholders(request, output_rows, file_block); + *rows = output_rows; + _pending_predicate_batch_rows_consumed = physical_end; + _pending_predicate_selected_offset = output_end; + if (_pending_predicate_selected_offset == _pending_predicate_selection.size()) { + DORIS_CHECK_EQ(_pending_predicate_batch_rows_consumed, _pending_predicate_batch_rows); + if (finish_current_reader_batch_profiles() && + _scan_profile.column_reader_profile.page_crossing_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.column_reader_profile.page_crossing_batches, 1); + } + _pending_predicate_batch_rows = 0; + _pending_predicate_batch_rows_consumed = 0; + _pending_predicate_selected_offset = 0; + _pending_predicate_selection.clear(); + _pending_predicate_columns.clear(); + _pending_output_selection.clear(); + } + return Status::OK(); +} + void ParquetScanScheduler::mark_condition_cache_granules(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_first_file_row) { @@ -1465,8 +2195,28 @@ Status ParquetScanScheduler::read_next_batch( const std::vector>& file_schema, const format::FileScanRequest& request, Block* file_block, size_t* rows, bool* eof) { *rows = 0; + if (!_pending_predicate_selection.empty()) { + RETURN_IF_ERROR(materialize_pending_predicate_batch(request, file_block, rows)); + *eof = false; + return Status::OK(); + } + int64_t predicate_batch_rows = _batch_size; + const int64_t max_predicate_batch_rows = std::min( + std::numeric_limits::max(), + std::max(DEFAULT_READ_BATCH_SIZE, _runtime_state == nullptr + ? DEFAULT_READ_BATCH_SIZE + : _runtime_state->batch_size())); + auto grow_empty_predicate_batch = [max_predicate_batch_rows](int64_t current) { + for (const int64_t target : + {int64_t {256}, int64_t {1024}, int64_t {4096}, max_predicate_batch_rows}) { + if (current < target) { + return std::min(target, max_predicate_batch_rows); + } + } + return max_predicate_batch_rows; + }; while (true) { - if (_current_row_group == nullptr) { + if (!_has_current_row_group) { bool has_row_group = false; RETURN_IF_ERROR( open_next_row_group(file_context, file_schema, request, &has_row_group)); @@ -1501,7 +2251,7 @@ Status ParquetScanScheduler::read_next_batch( continue; } - const int64_t batch_rows = std::min(_batch_size, remaining_rows); + const int64_t batch_rows = std::min(predicate_batch_rows, remaining_rows); const int64_t physical_rows_read = batch_rows; const int64_t batch_first_file_row = _current_row_group_first_row + _current_row_group_rows_read; @@ -1514,6 +2264,10 @@ Status ParquetScanScheduler::read_next_batch( _current_range_rows_read = 0; } if (*rows == 0) { + // Fully rejected probes carry no output-width sample. Widen predicate work to cross + // long empty prefixes cheaply; a later non-empty probe is sliced before lazy columns + // are materialized, so this internal width cannot escape the caller's row cap. + predicate_batch_rows = grow_empty_predicate_batch(predicate_batch_rows); continue; } *eof = false; diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index 3fa7586fb1b428..ac883cbbcbe099 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include @@ -33,13 +36,6 @@ #include "runtime/runtime_profile.h" #include "storage/segment/condition_cache.h" -namespace parquet { -class FileMetaData; -class ParquetFileReader; -class RowGroupMetaData; -class RowGroupReader; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -57,6 +53,42 @@ namespace doris::format::parquet { struct ParquetFileContext; struct ParquetColumnSchema; +struct ParquetPageCacheRange; +struct ParquetScanRange; +class NativeParquetMetadata; + +namespace detail { +struct PredicateConjunctSchedule { + std::map single_column_conjuncts; + VExprContextSPtrs remaining_conjuncts; +}; + +struct AdaptivePredicateStats { + double cost_per_input_row_ns = 0; + double survival_ratio = 1; + size_t samples = 0; +}; + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats); +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability); +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence); +Status validate_ephemeral_expr_result_column(size_t original_columns, int result_column_id, + size_t current_columns); +Status build_native_prefetch_ranges( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges); +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups); +} // namespace detail // ============================================================================ // ============================================================================ @@ -74,23 +106,39 @@ struct RowGroupReadPlan { std::vector selected_ranges; // row ranges to read after page-index pruning std::map page_skip_plans; // leaf_column_id -> data pages that can be skipped completely + // Deferred planning transfers parsed indexes to execution so narrowed scans never issue the + // same remote index reads a second time while opening the row group. + std::unordered_map offset_indexes; + // Footer statistics are cheap and eager. Remote dictionary/Bloom/page-index probes fill the + // remaining fields only when this row group reaches the scheduler. + bool expensive_pruning_pending = false; }; struct RowGroupScanPlan { std::vector row_groups; // row groups selected after pruning ParquetPruningStats pruning_stats; // pruning statistics + bool enable_bloom_filter = false; }; // ============================================================================ // ============================================================================ -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); + const RuntimeState* runtime_state = nullptr, + ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}); + +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile = nullptr); IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows); @@ -116,6 +164,9 @@ class ParquetScanScheduler { _page_skip_profile = page_skip_profile; } void set_scan_profile(ParquetScanProfile scan_profile) { _scan_profile = scan_profile; } + void set_pruning_profile(const ParquetProfile* parquet_profile) { + _parquet_profile = parquet_profile; + } void set_merge_read_options(RuntimeProfile* profile, int64_t merge_read_slice_size) { _profile = profile; _merge_read_slice_size = merge_read_slice_size; @@ -128,6 +179,10 @@ class ParquetScanScheduler { void set_enable_strict_mode(bool enable_strict_mode) { _enable_strict_mode = enable_strict_mode; } + void set_runtime_state(RuntimeState* runtime_state) { _runtime_state = runtime_state; } + // Release row-group readers before the owning RuntimeProfile is reported. Native readers + // publish their accumulated page/decode statistics from their destructor. + void close() { reset_current_row_group(); } // Upper scanner owns adaptive memory feedback; scheduler only applies the current row cap when // splitting selected row ranges into physical read batches. void set_batch_size(size_t batch_size) { @@ -145,13 +200,22 @@ class ParquetScanScheduler { bool* eof); private: + static constexpr size_t PROFILE_FLUSH_BATCH_INTERVAL = 16; + void reset_current_row_group(); + void flush_current_reader_profiles(); + bool finish_current_reader_batch_profiles(); + const detail::PredicateConjunctSchedule& predicate_conjunct_schedule( + const format::FileScanRequest& request); + std::vector adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const; Status open_next_row_group(ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, bool* has_row_group); Status skip_current_row_group_rows(int64_t rows); + Status flush_pending_non_predicate_skip_rows(); Status read_filter_columns(int64_t batch_rows, const format::FileScanRequest& request, Block* file_block, SelectionVector* selection, @@ -162,38 +226,40 @@ class ParquetScanScheduler { ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata); + const tparquet::RowGroup& row_group_metadata); - void prefetch_current_row_group_columns( + Status prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched); - bool prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx); - Status read_current_row_group_batch( ParquetFileContext& file_context, const std::vector>& file_schema, int64_t batch_rows, const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, size_t* rows); + Status materialize_pending_predicate_batch(const format::FileScanRequest& request, + Block* file_block, size_t* rows); + void mark_condition_cache_granules(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_first_file_row); std::vector _row_group_plans; // row group queue to scan size_t _next_row_group_plan_idx = 0; // index of the next row group to process - std::shared_ptr<::parquet::RowGroupReader> _current_row_group; // Arrow RowGroup reader - std::map> + bool _has_current_row_group = false; + // Readers retain pointers into this immutable row-group map, so it must outlive both maps below. + std::unordered_map _current_offset_indexes; + // File-local ids are signed because virtual columns use reserved negative values. Keeping the + // typed id as the map key prevents GLOBAL_ROWID_COLUMN_ID from wrapping to a storage ColumnId. + std::map> _current_predicate_columns; // predicate ColumnReaders - std::map> + std::map> _current_non_predicate_columns; // non-predicate ColumnReaders - std::map + std::map _current_dictionary_filters; // local id -> dict entry bitmap - std::map>> + std::map>> _current_dictionary_residual_conjuncts; // local id -> row-level residual conjuncts int64_t _current_row_group_rows = 0; // current row group row count int _current_row_group_id = -1; // current row group id in parquet metadata @@ -203,18 +269,50 @@ class ParquetScanScheduler { _current_selected_ranges; // selected ranges for the current row group after page-index pruning size_t _current_range_idx = 0; // current selected_range index int64_t _current_range_rows_read = 0; // rows read in the current range + // Predicate readers move immediately because they decide which rows survive. Non-predicate + // readers can lag behind across fully filtered batches and range gaps; the lag is flushed once + // before the next surviving batch is materialized, or discarded with the row group. + int64_t _pending_non_predicate_skip_rows = 0; + // Empty predicate batches may widen their physical probe. If the first non-empty probe finds + // more rows than the caller's cap, keep its narrow predicate result here and materialize lazy + // columns in capped physical slices on subsequent calls. + int64_t _pending_predicate_batch_rows = 0; + int64_t _pending_predicate_batch_rows_consumed = 0; + size_t _pending_predicate_selected_offset = 0; + std::vector _pending_predicate_selection; + std::map _pending_predicate_columns; + SelectionVector _pending_output_selection; bool _current_predicate_prefetched = false; bool _current_non_predicate_prefetched = false; bool _current_merge_range_active = false; ParquetPageSkipProfile _page_skip_profile; ParquetScanProfile _scan_profile; + const ParquetProfile* _parquet_profile = nullptr; RuntimeProfile* _profile = nullptr; int64_t _merge_read_slice_size = -1; std::optional _global_rowid_context; const cctz::time_zone* _timezone = nullptr; bool _enable_strict_mode = false; + bool _enable_bloom_filter = false; + RuntimeState* _runtime_state = nullptr; int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; + // Batch control scratch is scheduler-owned so adaptive row caps change logical sizes without + // reallocating selection indices, dense filter bytes, or compacted-column positions. + SelectionVector _selection; + std::vector _read_column_positions_scratch; + const format::FileScanRequest* _predicate_schedule_request = nullptr; + detail::PredicateConjunctSchedule _predicate_schedule; + std::vector _predicate_positions_scratch; + std::unordered_map _predicate_indices_by_position_scratch; + std::vector _ordered_predicate_positions_scratch; + std::unordered_map> + _predicate_column_selection_scratch; + IColumn::Filter _predicate_compaction_filter_scratch; + size_t _predicate_batch_sequence = 0; + size_t _batches_since_profile_flush = 0; + std::unordered_map _predicate_runtime_stats; + double _predicate_survival_ratio = -1; std::shared_ptr _condition_cache_ctx; int64_t _condition_cache_filtered_rows = 0; int64_t _predicate_filtered_rows = 0; diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index b1eba0cf74fe93..7d1afb39240ac1 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -15,15 +15,6 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include @@ -39,6 +30,7 @@ #include #include +#include "common/cast_set.h" #include "common/config.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_nullable.h" @@ -47,15 +39,99 @@ #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr_context.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/timestamp_statistics.h" #include "runtime/runtime_profile.h" +#include "storage/index/bloom_filter/bloom_filter.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/index/zone_map/zonemap_eval_context.h" +#include "util/thrift_util.h" +#include "util/unaligned.h" namespace doris::format::parquet { +namespace detail { + +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size) { + if (offset < 0 || header_size == 0 || payload_size < segment_v2::BloomFilter::MINIMUM_BYTES || + payload_size > segment_v2::BloomFilter::MAXIMUM_BYTES || payload_size % 32 != 0) { + return Status::Corruption( + "Invalid Parquet Bloom filter layout: offset {}, header {}, payload {}", offset, + header_size, payload_size); + } + const uint64_t unsigned_offset = static_cast(offset); + const uint64_t total_size = static_cast(header_size) + payload_size; + if (unsigned_offset > file_size || total_size > file_size - unsigned_offset) { + return Status::Corruption("Parquet Bloom filter range exceeds file size {}", file_size); + } + if (declared_length >= 0) { + const uint64_t unsigned_declared_length = static_cast(declared_length); + if (unsigned_declared_length < total_size || + unsigned_declared_length > file_size - unsigned_offset) { + return Status::Corruption( + "Parquet Bloom filter requires {} bytes, metadata declares {}, file has {}", + total_size, declared_length, file_size - unsigned_offset); + } + } + return Status::OK(); +} + +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id) { + return leaf_column_id >= 0 && metadata.__isset.column_orders && + leaf_column_id < static_cast(metadata.column_orders.size()) && + metadata.column_orders[leaf_column_id].__isset.TYPE_ORDER; +} + +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + auto sanitized = statistics; + if (!has_type_defined_order || !sanitized.__isset.min_value || !sanitized.__isset.max_value) { + sanitized.__isset.min_value = false; + sanitized.__isset.max_value = false; + sanitized.min_value.clear(); + sanitized.max_value.clear(); + } + const bool binary = type_descriptor.physical_type == tparquet::Type::BYTE_ARRAY || + type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; + if (!sanitized.__isset.min || !sanitized.__isset.max || + (binary && sanitized.min != sanitized.max)) { + sanitized.__isset.min = false; + sanitized.__isset.max = false; + sanitized.min.clear(); + sanitized.max.clear(); + } + return sanitized; +} + +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + // Inexact bounds remain useful for pruning, but returning them as aggregate values changes the + // query result. Missing exactness fields are legacy-compatible; only an explicit false rejects. + if ((statistics.__isset.is_min_value_exact && !statistics.is_min_value_exact) || + (statistics.__isset.is_max_value_exact && !statistics.is_max_value_exact)) { + return false; + } + const auto sanitized = + sanitize_native_footer_statistics(type_descriptor, statistics, has_type_defined_order); + return (sanitized.__isset.min_value && sanitized.__isset.max_value) || + (sanitized.__isset.min && sanitized.__isset.max); +} + +} // namespace detail + namespace { +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + int64_t page_rows, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone); + enum class ParquetRowGroupPruneReason { NONE, // cannot prune; must read STATISTICS, // excluded by ZoneMap statistics @@ -63,6 +139,60 @@ enum class ParquetRowGroupPruneReason { BLOOM_FILTER, // excluded by bloom filter }; +Status read_native_bloom_filter(const tparquet::ColumnMetaData& metadata, + const io::FileReaderSPtr& file, io::IOContext* io_ctx, + std::unique_ptr* result) { + if (result == nullptr || file == nullptr || !metadata.__isset.bloom_filter_offset) { + return Status::NotSupported("Parquet Bloom filter is unavailable"); + } + constexpr size_t MAX_BLOOM_HEADER_BYTES = 64; + if (metadata.bloom_filter_offset < 0 || + (metadata.__isset.bloom_filter_length && metadata.bloom_filter_length <= 0)) { + return Status::Corruption("Invalid Parquet Bloom filter offset or declared length"); + } + const uint64_t bloom_offset = static_cast(metadata.bloom_filter_offset); + if (bloom_offset >= file->size()) { + return Status::Corruption("Parquet Bloom filter offset exceeds file size {}", file->size()); + } + const size_t available = file->size() - bloom_offset; + const size_t declared_available = + metadata.__isset.bloom_filter_length + ? std::min(metadata.bloom_filter_length, available) + : available; + const size_t header_read_size = std::min(declared_available, MAX_BLOOM_HEADER_BYTES); + std::vector header_buffer(header_read_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(metadata.bloom_filter_offset, + Slice(header_buffer.data(), header_buffer.size()), &bytes_read, + io_ctx)); + tparquet::BloomFilterHeader header; + uint32_t header_size = cast_set(bytes_read); + RETURN_IF_ERROR(deserialize_thrift_msg(header_buffer.data(), &header_size, true, &header)); + if (!header.algorithm.__isset.BLOCK || !header.compression.__isset.UNCOMPRESSED || + !header.hash.__isset.XXHASH || header.numBytes <= 0) { + return Status::NotSupported("Unsupported Parquet Bloom filter encoding"); + } + + // Validate the complete split-block layout before allocating or adding footer-controlled + // offsets; BloomFilter::init() otherwise receives a truncated or oversized backing buffer. + RETURN_IF_ERROR(detail::validate_native_bloom_filter_layout( + metadata.bloom_filter_offset, header_size, header.numBytes, + metadata.__isset.bloom_filter_length ? metadata.bloom_filter_length : -1, + file->size())); + + std::vector data(cast_set(header.numBytes)); + RETURN_IF_ERROR(file->read_at(static_cast(metadata.bloom_filter_offset) + header_size, + Slice(data.data(), data.size()), &bytes_read, io_ctx)); + if (bytes_read != data.size()) { + return Status::Corruption("Truncated Parquet Bloom filter payload"); + } + auto bloom_filter = std::make_unique(); + RETURN_IF_ERROR(bloom_filter->init(reinterpret_cast(data.data()), data.size(), + segment_v2::HashStrategyPB::XX_HASH_64)); + *result = std::move(bloom_filter); + return Status::OK(); +} + bool bloom_logical_type_supported(const ParquetColumnSchema& column_schema) { if (column_schema.type == nullptr) { return false; @@ -108,8 +238,15 @@ Status read_decoded_field(const ParquetColumnSchema& column_schema, DecodedColum view.fixed_length = column_schema.type_descriptor.fixed_length; view.timestamp_is_adjusted_to_utc = column_schema.type_descriptor.timestamp_is_adjusted_to_utc; view.timezone = timezone; - return column_schema.type->get_serde()->read_field_from_decoded_value(*column_schema.type, - field, view); + // Statistics are pruning proofs, not row materialization. A malformed non-NULL bound must + // disable pruning instead of being converted to NULL under permissive scan semantics. + view.enable_strict_mode = true; + RETURN_IF_ERROR(column_schema.type->get_serde()->read_field_from_decoded_value( + *column_schema.type, field, view)); + if (field->is_null()) { + return Status::DataQualityError("Non-NULL Parquet statistic decoded as NULL"); + } + return Status::OK(); } template @@ -171,30 +308,6 @@ bool decoded_min_max_is_ordered(const ParquetColumnStatistics& column_statistics return !(column_statistics.max_value < column_statistics.min_value); } -template -bool set_decoded_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics>(statistics); - const auto& min_value = typed_statistics->min(); - const auto& max_value = typed_statistics->max(); - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; - } - } - if (!valid_min_max(min_value, max_value) || - !set_decoded_field(column_schema, value_kind, min_value, &column_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &column_statistics->max_value, - timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); -} - bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, const StringRef& value, Field* field, const cctz::time_zone* timezone) { @@ -205,54 +318,6 @@ bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedV return read_decoded_field(column_schema, view, field, timezone).ok(); } -bool set_string_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - switch (statistics->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::ByteArrayType>>( - statistics); - const auto min = ::parquet::ByteArrayToString(typed_statistics->min()); - const auto max = ::parquet::ByteArrayToString(typed_statistics->max()); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - if (column_schema.descriptor == nullptr || column_schema.descriptor->type_length() <= 0) { - return false; - } - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::FLBAType>>( - statistics); - const int type_length = column_schema.descriptor->type_length(); - const std::string min(reinterpret_cast(typed_statistics->min().ptr), - type_length); - const std::string max(reinterpret_cast(typed_statistics->max().ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - default: - return false; - } -} - template T load_predicate_value(const char* data) { T value; @@ -305,61 +370,19 @@ std::optional convert_logical_integer_to_physical_int32( return physical_value; } -class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { +class NativeParquetBloomFilterAdapter final : public segment_v2::BloomFilter { public: - ArrowParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, - const ::parquet::BloomFilter& bloom_filter) + NativeParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, + const segment_v2::BloomFilter& bloom_filter) : _column_schema(column_schema), _bloom_filter(bloom_filter) {} - void add_bytes(const char* buf, size_t size) override { DORIS_CHECK(false); } + void add_bytes(const char*, size_t) override { DORIS_CHECK(false); } bool test_bytes(const char* buf, size_t size) const override { - if (buf == nullptr) { - return true; - } - // Parquet bloom filters are populated from the physical column carrier, while VExpr - // literals are materialized as Doris logical values. Keep the logical type in - // BloomFilterEvalContext for expression compatibility, and normalize to the Parquet - // physical representation only at this adapter boundary. - switch (_column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - return test_boolean(buf, size); - case ::parquet::Type::INT32: - return test_physical_int32(buf, size); - case ::parquet::Type::INT64: - return test_int64(buf, size); - case ::parquet::Type::FLOAT: - return test_float(buf, size); - case ::parquet::Type::DOUBLE: - return test_double(buf, size); - case ::parquet::Type::BYTE_ARRAY: - return test_byte_array(buf, size); - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return test_fixed_len_byte_array(buf, size); - default: - return true; + if (buf == nullptr || + _column_schema.type_descriptor.physical_type != tparquet::Type::INT32) { + return _bloom_filter.test_bytes(buf, size); } - } - - void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } - bool has_null() const override { return false; } - void add_hash(uint64_t hash) override { DORIS_CHECK(false); } - bool test_hash(uint64_t hash) const override { return _bloom_filter.FindHash(hash); } - -private: - bool test_boolean(const char* buf, size_t size) const { - if (size == sizeof(bool)) { - const int32_t value = load_predicate_value(buf) ? 1 : 0; - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - if (size == sizeof(int32_t)) { - const int32_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value != 0 ? 1 : 0)); - } - return true; - } - - bool test_physical_int32(const char* buf, size_t size) const { const auto logical_value = load_predicate_integral_value(buf, size); if (!logical_value.has_value()) { return true; @@ -369,57 +392,20 @@ class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { if (!physical_value.has_value()) { return false; } - return find_int32(*physical_value); - } - - bool test_int64(const char* buf, size_t size) const { - if (size != sizeof(int64_t)) { - return true; - } - const int64_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_float(const char* buf, size_t size) const { - if (size != sizeof(float)) { - return true; - } - const float value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_double(const char* buf, size_t size) const { - if (size != sizeof(double)) { - return true; - } - const double value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_byte_array(const char* buf, size_t size) const { - ::parquet::ByteArray value(static_cast(size), - reinterpret_cast(buf)); - return _bloom_filter.FindHash(_bloom_filter.Hash(&value)); - } - - bool test_fixed_len_byte_array(const char* buf, size_t size) const { - if (_column_schema.type_descriptor.fixed_length <= 0) { - return true; - } - if (size != static_cast(_column_schema.type_descriptor.fixed_length)) { - return false; - } - ::parquet::FLBA value(reinterpret_cast(buf)); - return _bloom_filter.FindHash( - _bloom_filter.Hash(&value, _column_schema.type_descriptor.fixed_length)); + // Native file Bloom bytes are hashed from the Parquet physical carrier, not the wider + // Doris logical literal used by VExpr (for example UINT32 is exposed as BIGINT). + return _bloom_filter.test_bytes(reinterpret_cast(&*physical_value), + sizeof(*physical_value)); } - bool find_int32(int32_t value) const { - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } + void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } + bool has_null() const override { return false; } + void add_hash(uint64_t) override { DORIS_CHECK(false); } + bool test_hash(uint64_t hash) const override { return _bloom_filter.test_hash(hash); } +private: const ParquetColumnSchema& _column_schema; - const ::parquet::BloomFilter& _bloom_filter; + const segment_v2::BloomFilter& _bloom_filter; }; bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { @@ -427,14 +413,14 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { return false; } switch (column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BOOLEAN: + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::BYTE_ARRAY: return true; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return column_schema.type_descriptor.is_string_like && column_schema.type_descriptor.fixed_length > 0; default: @@ -442,220 +428,6 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { } } -bool bloom_filter_excludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - if (!bloom_filter_supported(column_schema)) { - return false; - } - ArrowParquetBloomFilterAdapter adapter(column_schema, bloom_filter); - BloomFilterEvalContext ctx; - ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { - .data_type = column_schema.type, - .bloom_filter = &adapter, - }); - return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; -} - -struct RowGroupBloomFilterCache { - using CacheKey = std::pair; - - ::parquet::BloomFilterReader* bloom_filter_reader = nullptr; - std::map> column_bloom_filters; - std::set loaded_columns; - - ::parquet::BloomFilter* get(int row_group_idx, int leaf_column_id, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_reader == nullptr || leaf_column_id < 0) { - return nullptr; - } - const CacheKey cache_key {row_group_idx, leaf_column_id}; - if (loaded_columns.find(cache_key) == loaded_columns.end()) { - loaded_columns.insert(cache_key); - try { - std::shared_ptr<::parquet::RowGroupBloomFilterReader> row_group_reader; - if (pruning_stats != nullptr) { - SCOPED_RAW_TIMER(&pruning_stats->bloom_filter_read_time); - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } else { - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } - } catch (const ::parquet::ParquetException&) { - return nullptr; - } catch (const std::exception&) { - return nullptr; - } - } - auto it = column_bloom_filters.find(cache_key); - return it == column_bloom_filters.end() ? nullptr : it->second.get(); - } -}; - -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; -} - -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; -} - -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; -} - -bool is_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { - return false; - } - - const auto& encoding_stats = column_metadata.encoding_stats(); - if (!encoding_stats.empty()) { - bool has_dictionary_data_page = false; - for (const auto& encoding_stat : encoding_stats) { - if (!is_data_page_type(encoding_stat.page_type) || encoding_stat.count <= 0) { - continue; - } - if (!is_dictionary_data_encoding(encoding_stat.encoding)) { - return false; - } - has_dictionary_data_page = true; - } - return has_dictionary_data_page; - } - - bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { - if (is_dictionary_data_encoding(encoding)) { - has_dictionary_encoding = true; - continue; - } - if (!is_level_encoding(encoding)) { - return false; - } - } - return has_dictionary_encoding; -} - -bool supports_dictionary_pruning(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr) { - return false; - } - if (!column_schema.type_descriptor.is_string_like) { - return false; - } - if (column_metadata.type() != ::parquet::Type::BYTE_ARRAY && - column_metadata.type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - return false; - } - return true; -} - -} // namespace - -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words) { - DORIS_CHECK(dict_words != nullptr); - dict_words->clear(); - if (file_reader == nullptr || leaf_column_id < 0) { - return false; - } - - auto row_group_reader = file_reader->RowGroup(row_group_idx); - if (row_group_reader == nullptr) { - return false; - } - auto page_reader = row_group_reader->GetColumnPageReader(leaf_column_id); - if (page_reader == nullptr) { - return false; - } - - std::shared_ptr<::parquet::Page> page; - try { - page = page_reader->NextPage(); - } catch (const ::parquet::ParquetException&) { - return false; - } catch (const std::exception&) { - return false; - } - if (page == nullptr || page->type() != ::parquet::PageType::DICTIONARY_PAGE) { - return false; - } - const auto* dictionary_page = static_cast(page.get()); - if (dictionary_page->encoding() != ::parquet::Encoding::PLAIN && - dictionary_page->encoding() != ::parquet::Encoding::PLAIN_DICTIONARY) { - return false; - } - const int32_t dictionary_length = dictionary_page->num_values(); - if (dictionary_length <= 0) { - return false; - } - const auto* dictionary_data = dictionary_page->data(); - const int dictionary_size = dictionary_page->size(); - - dict_words->values.reserve(static_cast(dictionary_length)); - if (column_schema.descriptor->physical_type() == ::parquet::Type::BYTE_ARRAY) { - auto decoder = ::parquet::MakeTypedDecoder<::parquet::ByteArrayType>( - ::parquet::Encoding::PLAIN, column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::ByteArray> byte_array_values(static_cast(dictionary_length)); - if (decoder->Decode(byte_array_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(byte_array_values[dict_idx].ptr), - byte_array_values[dict_idx].len); - } - dict_words->build_refs(); - return true; - } - if (column_schema.descriptor->physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; - } - auto decoder = ::parquet::MakeTypedDecoder<::parquet::FLBAType>(::parquet::Encoding::PLAIN, - column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::FixedLenByteArray> flba_values( - static_cast(dictionary_length)); - if (decoder->Decode(flba_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(flba_values[dict_idx].ptr), type_length); - } - dict_words->build_refs(); - return true; - } - return false; -} - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words) { - std::vector fields; - fields.reserve(dict_words.refs.size()); - for (const auto& ref : dict_words.refs) { - fields.push_back(Field::create_field(String(ref.data, ref.size))); - } - return fields; -} - -namespace { - const ParquetColumnSchema* resolve_local_leaf_schema( const std::vector>& schema, const format::LocalColumnId file_column_id) { @@ -761,164 +533,112 @@ void accumulate_zonemap_stats(const ZoneMapEvalContext& ctx, ParquetPruningStats } // namespace +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state) { + return config::enable_parquet_page_index && has_expr_zonemap_filter(request, runtime_state); +} + std::shared_ptr ParquetStatisticsUtils::MakeZoneMap( const ParquetColumnStatistics& statistics) { return make_zonemap_from_statistics(statistics); } ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, const cctz::time_zone* timezone) { + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone) { ParquetColumnStatistics result; - if (statistics == nullptr) { + if (statistics == nullptr || column_value_count < 0) { return result; } - result.has_null = !statistics->HasNullCount() || statistics->null_count() > 0; - result.has_not_null = statistics->num_values() > 0 || statistics->HasMinMax(); - result.has_null_count = statistics->HasNullCount(); - if (!result.has_not_null || !statistics->HasMinMax()) { + if (statistics->__isset.null_count && statistics->null_count > column_value_count) { + // An impossible null count makes all derived min/max and all-null flags untrustworthy; + // disable pruning instead of turning corrupt footer metadata into false negatives. return result; } - DORIS_CHECK(column_schema.type != nullptr); - switch (statistics->physical_type()) { - case ::parquet::Type::BOOLEAN: - result.has_min_max = set_decoded_min_max<::parquet::BooleanType>( - statistics, column_schema, DecodedValueKind::BOOL, &result, timezone); - return result; - case ::parquet::Type::INT32: - result.has_min_max = set_decoded_min_max<::parquet::Int32Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::INT64: - result.has_min_max = set_decoded_min_max<::parquet::Int64Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::FLOAT: - result.has_min_max = set_decoded_min_max<::parquet::FloatType>( - statistics, column_schema, DecodedValueKind::FLOAT, &result, timezone); - return result; - case ::parquet::Type::DOUBLE: - result.has_min_max = set_decoded_min_max<::parquet::DoubleType>( - statistics, column_schema, DecodedValueKind::DOUBLE, &result, timezone); - return result; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result.has_min_max = set_string_min_max(statistics, column_schema, &result, timezone); - return result; - default: - return result; + const bool has_null_count = statistics->__isset.null_count && statistics->null_count >= 0; + const int64_t null_count = has_null_count ? statistics->null_count : 0; + const bool has_not_null = has_null_count ? column_value_count > null_count : true; + const std::string* min_value = statistics->__isset.min_value + ? &statistics->min_value + : (statistics->__isset.min ? &statistics->min : nullptr); + const std::string* max_value = statistics->__isset.max_value + ? &statistics->max_value + : (statistics->__isset.max ? &statistics->max : nullptr); + + tparquet::ColumnIndex index; + index.__set_null_pages({!has_not_null}); + index.__set_null_counts({null_count}); + if (min_value != nullptr && max_value != nullptr) { + index.__set_min_values({*min_value}); + index.__set_max_values({*max_value}); + } + // Footer statistics and page indexes share the same little-endian physical encoding. Reusing + // one decoder keeps native row-group and page pruning identical for logical types and NaNs. + if (!build_native_page_statistics(index, column_schema, 0, column_value_count, &result, + timezone)) { + return {}; + } + if (!has_null_count) { + result.has_null_count = false; + result.has_null = true; } + return result; } -bool ParquetStatisticsUtils::BloomFilterExcludes(const ParquetColumnSchema& column_schema, - int slot_index, const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - return bloom_filter_excludes(column_schema, slot_index, conjuncts, bloom_filter); +bool ParquetStatisticsUtils::NativeBloomFilterExcludes( + const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, const segment_v2::BloomFilter& bloom_filter) { + if (!bloom_filter_supported(column_schema)) { + return false; + } + NativeParquetBloomFilterAdapter adapter(column_schema, bloom_filter); + BloomFilterEvalContext ctx; + ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { + .data_type = column_schema.type, + .bloom_filter = &adapter, + }); + return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; } namespace { -ParquetRowGroupPruneReason dictionary_prune_reason( - const ::parquet::RowGroupMetaData& row_group, ::parquet::ParquetFileReader* file_reader, - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request) { - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_dictionary_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { - continue; - } - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_dictionary_pruning(*column_schema, *column_chunk) || - !is_dictionary_encoded_chunk(*column_chunk)) { - continue; - } - - ParquetDictionaryWords dict_words; - if (!read_dictionary_words(file_reader, row_group_idx, column_schema->leaf_column_id, - *column_schema, &dict_words)) { - continue; - } - DictionaryEvalContext ctx; - ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { - .data_type = column_schema->type, - .values = dictionary_fields_from_words(dict_words), - }); - if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch) { - return ParquetRowGroupPruneReason::DICTIONARY; +void collect_filtered_leaf_ids(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::set* leaf_column_ids) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (column_schema.leaf_column_id >= 0) { + leaf_column_ids->insert(column_schema.leaf_column_id); } + return; } - return ParquetRowGroupPruneReason::NONE; -} - -ParquetRowGroupPruneReason bloom_filter_prune_reason( - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request, RowGroupBloomFilterCache* bloom_filter_cache, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_cache == nullptr) { - return ParquetRowGroupPruneReason::NONE; - } - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr || - !bloom_filter_supported(*column_schema)) { - continue; - } - auto* bloom_filter = bloom_filter_cache->get(row_group_idx, column_schema->leaf_column_id, - pruning_stats); - if (bloom_filter == nullptr) { + for (const auto& child_schema : column_schema.children) { + if (!format::is_child_projected(projection, child_schema->local_id)) { continue; } - if (ParquetStatisticsUtils::BloomFilterExcludes(*column_schema, slot_index, conjuncts, - *bloom_filter)) { - return ParquetRowGroupPruneReason::BLOOM_FILTER; - } + collect_filtered_leaf_ids(*child_schema, + format::find_child_projection(projection, child_schema->local_id), + leaf_column_ids); } - return ParquetRowGroupPruneReason::NONE; } -void init_bloom_filter_cache(::parquet::ParquetFileReader* file_reader, bool enable_bloom_filter, - RowGroupBloomFilterCache* bloom_filter_cache) { - DORIS_CHECK(bloom_filter_cache != nullptr); - if (!enable_bloom_filter || file_reader == nullptr) { - return; - } - try { - bloom_filter_cache->bloom_filter_reader = &file_reader->GetBloomFilterReader(); - } catch (const ::parquet::ParquetException&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } catch (const std::exception&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } +bool native_metadata_predicate_is_type_safe(const ParquetColumnSchema& column_schema) { + DORIS_CHECK(column_schema.type != nullptr); + // Raw VARBINARY file slots may feed table-side STRING casts. Footer/page metadata is still in + // the pre-cast domain, so using it for a rewritten table predicate can cause false negatives. + return remove_nullable(column_schema.type)->get_primitive_type() != TYPE_VARBINARY; } -bool check_statistics(const ::parquet::RowGroupMetaData& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { +bool check_native_statistics(const tparquet::FileMetaData& metadata, + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) { const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts); if (slot_indexes.empty()) { return false; } - ZoneMapEvalContext ctx; for (const int slot_index : slot_indexes) { const auto file_column_id = file_column_id_by_block_position(request, slot_index); @@ -926,267 +646,320 @@ bool check_statistics(const ::parquet::RowGroupMetaData& row_group, continue; } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { continue; } - + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; std::shared_ptr zone_map; - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk != nullptr) { + if (chunk.__isset.meta_data) { + const auto& column_metadata = chunk.meta_data; + std::optional safe_statistics; + if (column_metadata.__isset.statistics) { + safe_statistics = detail::sanitize_native_footer_statistics( + column_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order(metadata, + column_schema->leaf_column_id)); + } zone_map = ParquetStatisticsUtils::MakeZoneMap( ParquetStatisticsUtils::TransformColumnStatistics( - *column_schema, column_chunk->statistics(), timezone)); + *column_schema, + safe_statistics.has_value() ? &*safe_statistics : nullptr, + column_metadata.num_values, timezone)); } add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map)); } - const auto result = VExprContext::evaluate_zonemap_filter(request.conjuncts, ctx); accumulate_zonemap_stats(ctx, pruning_stats); return result == ZoneMapFilterResult::kNoMatch; } -Status select_row_groups_by_metadata_impl( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - int64_t row_group_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &row_group_filter_time_sink - : &pruning_stats->row_group_filter_time); - if (selected_row_groups == nullptr) { - return Status::InvalidArgument("selected_row_groups is null"); - } - selected_row_groups->clear(); +bool is_native_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; +} - const int num_row_groups = metadata.num_row_groups(); - if (pruning_stats != nullptr) { - pruning_stats->total_row_groups = num_row_groups; +bool is_native_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; +} + +bool is_native_dictionary_encoded_chunk(const tparquet::ColumnMetaData& metadata) { + if (!metadata.__isset.dictionary_page_offset || metadata.dictionary_page_offset < 0) { + return false; } - const auto candidate_size = candidate_row_groups == nullptr - ? static_cast(num_row_groups) - : candidate_row_groups->size(); - selected_row_groups->reserve(candidate_size); - RowGroupBloomFilterCache bloom_filter_cache; - init_bloom_filter_cache(file_reader, enable_bloom_filter, &bloom_filter_cache); - for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { - const int row_group_idx = candidate_row_groups == nullptr - ? static_cast(candidate_idx) - : (*candidate_row_groups)[candidate_idx]; - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(row_group_idx < num_row_groups); - auto row_group = metadata.RowGroup(row_group_idx); - if (row_group == nullptr) { - selected_row_groups->push_back(row_group_idx); - continue; + if (metadata.__isset.encoding_stats && !metadata.encoding_stats.empty()) { + bool has_dictionary_data_page = false; + for (const auto& encoding_stat : metadata.encoding_stats) { + if ((encoding_stat.page_type != tparquet::PageType::DATA_PAGE && + encoding_stat.page_type != tparquet::PageType::DATA_PAGE_V2) || + encoding_stat.count <= 0) { + continue; + } + if (!is_native_dictionary_data_encoding(encoding_stat.encoding)) { + return false; + } + has_dictionary_data_page = true; } - ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; - if (has_expr_zonemap_filter(request, runtime_state) && - check_statistics(*row_group, file_schema, request, pruning_stats, timezone)) { - prune_reason = ParquetRowGroupPruneReason::STATISTICS; + return has_dictionary_data_page; + } + bool has_dictionary_encoding = false; + for (const auto encoding : metadata.encodings) { + if (is_native_dictionary_data_encoding(encoding)) { + has_dictionary_encoding = true; + } else if (!is_native_level_encoding(encoding)) { + return false; } + } + return has_dictionary_encoding; +} - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = dictionary_prune_reason(*row_group, file_reader, row_group_idx, - file_schema, request); - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = bloom_filter_prune_reason(row_group_idx, file_schema, request, - &bloom_filter_cache, pruning_stats); - } +const format::LocalColumnIndex* find_request_projection(const format::FileScanRequest& request, + format::LocalColumnId file_column_id) { + for (const auto& projection : request.predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; } - - if (prune_reason != ParquetRowGroupPruneReason::NONE) { - if (pruning_stats != nullptr) { - pruning_stats->filtered_group_rows += row_group->num_rows(); - if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { - ++pruning_stats->filtered_row_groups_by_statistics; - } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { - ++pruning_stats->filtered_row_groups_by_dictionary; - } else if (prune_reason == ParquetRowGroupPruneReason::BLOOM_FILTER) { - ++pruning_stats->filtered_row_groups_by_bloom_filter; - } - } - continue; + } + for (const auto& projection : request.non_predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; } - selected_row_groups->push_back(row_group_idx); } - return Status::OK(); + return nullptr; } -} // namespace - -Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, +ParquetRowGroupPruneReason native_dictionary_prune_reason( + const tparquet::RowGroup& row_group, int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - return select_row_groups_by_metadata_impl( - metadata, file_reader, file_schema, request, candidate_row_groups, selected_row_groups, - enable_bloom_filter, pruning_stats, timezone, runtime_state); -} - -namespace { - -template -bool set_page_decoded_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - DecodedValueKind value_kind, ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - const auto typed_index = - std::static_pointer_cast<::parquet::TypedColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; + const format::FileScanRequest& request, const cctz::time_zone* timezone, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile) { + if (file_context == nullptr || file_context->native_metadata == nullptr) { + return ParquetRowGroupPruneReason::NONE; } - const auto& min_value = typed_index->min_values()[page_idx]; - const auto& max_value = typed_index->max_values()[page_idx]; - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_dictionary_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + const auto* projection = find_request_projection(request, *file_column_id); + if (column_schema == nullptr || projection == nullptr || column_schema->type == nullptr || + !column_schema->type_descriptor.is_string_like || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + if (!native_metadata_predicate_is_type_safe(*column_schema)) { + // The file-local VARBINARY may feed a table-side STRING cast. Pruning before that cast + // can compare different Field kinds and incorrectly discard a matching row group. + continue; + } + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data || + (chunk.meta_data.type != tparquet::Type::BYTE_ARRAY && + chunk.meta_data.type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) || + !is_native_dictionary_encoded_chunk(chunk.meta_data)) { + continue; + } + std::unique_ptr reader; + const std::vector ranges {{0, row_group.num_rows}}; + const std::unordered_map offset_indexes; + // Metadata pruning uses the real native reader, so its page work must be attributed to the + // scan profile even when the row group is eliminated before execution readers are built. + const auto status = NativeColumnReader::create( + *column_schema, projection, file_context->native_file, + file_context->native_metadata, row_group_idx, ranges, offset_indexes, timezone, + file_context->native_io_ctx, nullptr, file_context->native_page_cache_enabled, + file_context->native_page_cache_file_key, true, column_reader_profile, &reader); + if (!status.ok() || reader == nullptr) { + continue; + } + auto dictionary_result = reader->dictionary_values(); + if (!dictionary_result.has_value()) { + continue; + } + auto dictionary = std::move(dictionary_result).value(); + std::vector values(dictionary->size()); + for (size_t value_idx = 0; value_idx < dictionary->size(); ++value_idx) { + dictionary->get(value_idx, values[value_idx]); + } + DictionaryEvalContext ctx; + ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { + .data_type = column_schema->type, + .values = std::move(values), + }); + if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == + ZoneMapFilterResult::kNoMatch) { + return ParquetRowGroupPruneReason::DICTIONARY; } } - if (!valid_min_max(min_value, max_value)) { - // A NaN invalidates only this page's bounds, not the ColumnIndex itself. Keep the page - // conservatively by returning usable null-count statistics with has_min_max=false, while - // allowing later pages with finite bounds to remain eligible for pruning. - return true; - } - if (!set_decoded_field(column_schema, value_kind, min_value, &page_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &page_statistics->max_value, - timezone)) { - return false; - } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; - } - page_statistics->has_min_max = true; - return true; + return ParquetRowGroupPruneReason::NONE; } -bool set_page_string_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: { - const auto typed_index = - std::static_pointer_cast<::parquet::ByteArrayColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; - } - const auto min = ::parquet::ByteArrayToString(typed_index->min_values()[page_idx]); - const auto max = ::parquet::ByteArrayToString(typed_index->max_values()[page_idx]); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; +ParquetRowGroupPruneReason native_bloom_filter_prune_reason( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, ParquetFileContext* file_context, + ParquetPruningStats* pruning_stats) { + if (file_context == nullptr || file_context->native_file == nullptr) { + return ParquetRowGroupPruneReason::NONE; + } + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + !bloom_filter_supported(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; } - page_statistics->has_min_max = true; - return true; - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data) { + continue; } - const auto typed_index = std::static_pointer_cast<::parquet::FLBAColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; + std::unique_ptr bloom_filter; + Status status; + { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->bloom_filter_read_time); + status = read_native_bloom_filter(chunk.meta_data, file_context->native_file, + file_context->native_io_ctx, &bloom_filter); } - const std::string min( - reinterpret_cast(typed_index->min_values()[page_idx].ptr), - type_length); - const std::string max( - reinterpret_cast(typed_index->max_values()[page_idx].ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; + if (!status.ok() || bloom_filter == nullptr) { + continue; } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; + if (ParquetStatisticsUtils::NativeBloomFilterExcludes(*column_schema, slot_index, conjuncts, + *bloom_filter)) { + return ParquetRowGroupPruneReason::BLOOM_FILTER; } - page_statistics->has_min_max = true; - return true; - } - default: - return false; } + return ParquetRowGroupPruneReason::NONE; } -bool set_page_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - DORIS_CHECK(column_schema.type != nullptr); - switch (column_schema.descriptor->physical_type()) { - case ::parquet::Type::BOOLEAN: - return set_page_decoded_min_max<::parquet::BooleanType>(column_index, column_schema, - page_idx, DecodedValueKind::BOOL, - page_statistics, timezone); - case ::parquet::Type::INT32: - return set_page_decoded_min_max<::parquet::Int32Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::INT64: - return set_page_decoded_min_max<::parquet::Int64Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::FLOAT: - return set_page_decoded_min_max<::parquet::FloatType>(column_index, column_schema, page_idx, - DecodedValueKind::FLOAT, - page_statistics, timezone); - case ::parquet::Type::DOUBLE: - return set_page_decoded_min_max<::parquet::DoubleType>(column_index, column_schema, - page_idx, DecodedValueKind::DOUBLE, - page_statistics, timezone); - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return set_page_string_min_max(column_index, column_schema, page_idx, page_statistics, - timezone); - default: - return false; +int64_t native_requested_compressed_bytes( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request) { + std::set leaf_column_ids; + auto collect_projection = [&](const format::LocalColumnIndex& projection) { + const int32_t local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return; + } + collect_filtered_leaf_ids(*file_schema[local_id], &projection, &leaf_column_ids); + }; + for (const auto& projection : request.predicate_columns) { + collect_projection(projection); + } + for (const auto& projection : request.non_predicate_columns) { + collect_projection(projection); + } + int64_t bytes = 0; + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (chunk.__isset.meta_data && chunk.meta_data.total_compressed_size > 0) { + bytes += chunk.meta_data.total_compressed_size; + } } + return bytes; } -bool build_page_statistics(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - DORIS_CHECK(page_statistics != nullptr); - *page_statistics = ParquetColumnStatistics {}; +} // namespace - const auto& null_pages = column_index->null_pages(); - if (!column_index->has_null_counts() || page_idx >= null_pages.size() || - page_idx >= column_index->null_counts().size()) { - return false; +Status select_row_groups_by_metadata( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector* candidate_row_groups, + std::vector* selected_row_groups, bool enable_bloom_filter, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile, + ParquetMetadataProbeMode probe_mode) { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->row_group_filter_time); + if (selected_row_groups == nullptr) { + return Status::InvalidArgument("selected_row_groups is null"); } - - page_statistics->has_null_count = true; - page_statistics->has_null = column_index->null_counts()[page_idx] > 0; - page_statistics->has_not_null = !null_pages[page_idx]; - if (!page_statistics->has_not_null) { - return true; + selected_row_groups->clear(); + const size_t candidate_size = candidate_row_groups == nullptr ? metadata.row_groups.size() + : candidate_row_groups->size(); + if (pruning_stats != nullptr) { + pruning_stats->total_row_groups = cast_set(candidate_size); } - return set_page_min_max(column_index, column_schema, page_idx, page_statistics, timezone); + selected_row_groups->reserve(candidate_size); + for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { + const int row_group_idx = candidate_row_groups == nullptr + ? static_cast(candidate_idx) + : (*candidate_row_groups)[candidate_idx]; + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + // Candidate ids originate in external split metadata; a corrupt id must not terminate + // the BE while planning an otherwise recoverable file scan. + return Status::Corruption("Invalid Parquet row group candidate {} for {} row groups", + row_group_idx, metadata.row_groups.size()); + } + const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.num_rows == 0) { + // Native metadata probes construct positive row ranges; empty groups contribute no + // rows and must be discarded before dictionary, statistics, or Bloom reader setup. + continue; + } + ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; + if (probe_mode != ParquetMetadataProbeMode::EXPENSIVE_ONLY && + has_expr_zonemap_filter(request, runtime_state) && + check_native_statistics(metadata, row_group, file_schema, request, pruning_stats, + timezone)) { + prune_reason = ParquetRowGroupPruneReason::STATISTICS; + } + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE) { + prune_reason = + native_dictionary_prune_reason(row_group, row_group_idx, file_schema, request, + timezone, file_context, column_reader_profile); + } + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE && enable_bloom_filter) { + prune_reason = native_bloom_filter_prune_reason(row_group, file_schema, request, + file_context, pruning_stats); + } + if (prune_reason == ParquetRowGroupPruneReason::NONE) { + selected_row_groups->push_back(row_group_idx); + continue; + } + if (pruning_stats != nullptr) { + pruning_stats->filtered_group_rows += row_group.num_rows; + pruning_stats->filtered_bytes += + native_requested_compressed_bytes(row_group, file_schema, request); + if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { + ++pruning_stats->filtered_row_groups_by_statistics; + } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { + ++pruning_stats->filtered_row_groups_by_dictionary; + } else { + ++pruning_stats->filtered_row_groups_by_bloom_filter; + } + } + } + return Status::OK(); } +namespace { + std::vector intersect_ranges(const std::vector& left, const std::vector& right) { std::vector result; @@ -1219,19 +992,6 @@ int64_t count_range_rows(const std::vector& ranges) { return rows; } -RowRange page_row_range(const ::parquet::OffsetIndex& offset_index, size_t page_idx, - int64_t row_group_rows) { - const auto& page_locations = offset_index.page_locations(); - const int64_t start = page_locations[page_idx].first_row_index; - const int64_t end = page_idx + 1 == page_locations.size() - ? row_group_rows - : page_locations[page_idx + 1].first_row_index; - DORIS_CHECK(start >= 0); - DORIS_CHECK(end >= start); - DORIS_CHECK(end <= row_group_rows); - return RowRange {start, end - start}; -} - void append_row_range(const RowRange& range, std::vector* ranges) { if (range.length == 0) { return; @@ -1246,85 +1006,6 @@ void append_row_range(const RowRange& range, std::vector* ranges) { ranges->push_back(range); } -std::optional< - std::pair, std::shared_ptr<::parquet::OffsetIndex>>> -load_page_indexes_for_slot(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, - const ParquetColumnSchema** column_schema) { - DORIS_CHECK(column_schema != nullptr); - *column_schema = nullptr; - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - return std::nullopt; - } - *column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (*column_schema == nullptr || (*column_schema)->descriptor == nullptr) { - return std::nullopt; - } - - try { - auto column_index = row_group->GetColumnIndex((*column_schema)->leaf_column_id); - auto offset_index = row_group->GetOffsetIndex((*column_schema)->leaf_column_id); - if (column_index == nullptr || offset_index == nullptr || - column_index->null_pages().size() != offset_index->page_locations().size()) { - return std::nullopt; - } - return std::make_pair(std::move(column_index), std::move(offset_index)); - } catch (const ::parquet::ParquetException&) { - return std::nullopt; - } catch (const std::exception&) { - return std::nullopt; - } -} - -bool select_ranges_for_expr_zonemap( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, const VExprContextSPtrs& conjuncts, - int64_t row_group_rows, std::vector* ranges, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { - DORIS_CHECK(ranges != nullptr); - if (conjuncts.empty()) { - return false; - } - const ParquetColumnSchema* column_schema = nullptr; - const auto page_indexes = - load_page_indexes_for_slot(row_group, file_schema, request, slot_index, &column_schema); - if (!page_indexes.has_value()) { - return false; - } - const auto& [column_index, offset_index] = *page_indexes; - - ranges->clear(); - ZoneMapEvalStats page_stats; - const auto page_count = offset_index->page_locations().size(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - ParquetColumnStatistics page_statistics; - if (!ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *column_schema, page_idx, &page_statistics, timezone)) { - ranges->clear(); - return false; - } - - ZoneMapEvalContext ctx; - add_slot_zonemap(&ctx, slot_index, column_schema->type, - ParquetStatisticsUtils::MakeZoneMap(page_statistics)); - const auto result = VExprContext::evaluate_zonemap_filter(conjuncts, ctx); - page_stats.merge_page_eval_stats(ctx.stats); - if (result == ZoneMapFilterResult::kNoMatch) { - continue; - } - append_row_range(page_row_range(*offset_index, page_idx, row_group_rows), ranges); - } - if (pruning_stats != nullptr) { - pruning_stats->expr_zonemap_unusable_evals += page_stats.unusable_zonemap_eval_count; - pruning_stats->in_zonemap_point_check_count += page_stats.in_zonemap_point_check_count; - pruning_stats->in_zonemap_range_only_count += page_stats.in_zonemap_range_only_count; - } - return true; -} - bool ranges_intersect(const std::vector& ranges, const RowRange& range) { const int64_t range_end = range.start + range.length; for (const auto& selected_range : ranges) { @@ -1384,127 +1065,201 @@ void collect_request_leaf_schemas( } } -bool build_page_skip_plan_for_leaf( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const ParquetColumnSchema& column_schema, const std::vector& selected_ranges, - int64_t row_group_rows, ParquetPageSkipPlan* page_skip_plan) { - DORIS_CHECK(page_skip_plan != nullptr); - *page_skip_plan = ParquetPageSkipPlan {}; - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.leaf_column_id < 0 || - column_schema.descriptor->max_repetition_level() != 0) { +template +bool set_native_page_scalar_min_max(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + DecodedValueKind kind, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + if (page_idx >= column_index.min_values.size() || page_idx >= column_index.max_values.size() || + column_index.min_values[page_idx].size() != sizeof(ValueType) || + column_index.max_values[page_idx].size() != sizeof(ValueType)) { return false; } - - std::shared_ptr<::parquet::OffsetIndex> offset_index; - try { - offset_index = row_group->GetOffsetIndex(column_schema.leaf_column_id); - } catch (const ::parquet::ParquetException&) { - return false; - } catch (const std::exception&) { - return false; + const auto min_value = unaligned_load(column_index.min_values[page_idx].data()); + const auto max_value = unaligned_load(column_index.max_values[page_idx].data()); + if constexpr (std::is_integral_v) { + if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_TIMEV2) { + int64_t units_per_day = 0; + switch (column_schema.type_descriptor.time_unit) { + case ParquetTimeUnit::MILLIS: + units_per_day = 86400000; + break; + case ParquetTimeUnit::MICROS: + units_per_day = 86400000000; + break; + case ParquetTimeUnit::NANOS: + units_per_day = 86400000000000; + break; + default: + return false; + } + // TIME statistics are pruning proofs. Validate the raw carrier before rescaling so an + // invalid bound at or beyond 24:00 cannot publish a misleading ZoneMap. + if (min_value < 0 || max_value < 0 || min_value >= units_per_day || + max_value >= units_per_day) { + return false; + } + } + } + if constexpr (std::is_same_v) { + if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { + return false; + } + } + if (!valid_min_max(min_value, max_value)) { + return true; } - if (offset_index == nullptr) { + if (!set_decoded_field(column_schema, kind, min_value, &page_statistics->min_value, timezone) || + !set_decoded_field(column_schema, kind, max_value, &page_statistics->max_value, timezone)) { return false; } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; +} - const auto page_count = offset_index->page_locations().size(); - page_skip_plan->leaf_column_id = column_schema.leaf_column_id; - page_skip_plan->skipped_pages.resize(page_count); - page_skip_plan->skipped_page_compressed_sizes.resize(page_count); - const auto& page_locations = offset_index->page_locations(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - const RowRange row_range = page_row_range(*offset_index, page_idx, row_group_rows); - if (row_range.length == 0 || ranges_intersect(selected_ranges, row_range)) { - continue; - } - page_skip_plan->skipped_pages[page_idx] = 1; - page_skip_plan->skipped_page_compressed_sizes[page_idx] = - page_locations[page_idx].compressed_page_size; - append_row_range(row_range, &page_skip_plan->skipped_ranges); +bool set_native_page_boolean_min_max(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + if (page_idx >= column_index.min_values.size() || page_idx >= column_index.max_values.size() || + column_index.min_values[page_idx].size() != 1 || + column_index.max_values[page_idx].size() != 1) { + return false; } - if (page_skip_plan->empty()) { - *page_skip_plan = ParquetPageSkipPlan {}; + // Parquet BOOLEAN statistics use the same one-bit value representation as PLAIN pages; bits + // outside the value bit are padding and must not change false into true. + const uint8_t min_value = static_cast(column_index.min_values[page_idx][0]) & 1; + const uint8_t max_value = static_cast(column_index.max_values[page_idx][0]) & 1; + if (!valid_min_max(min_value, max_value)) { + return true; + } + if (!set_decoded_field(column_schema, DecodedValueKind::BOOL, min_value, + &page_statistics->min_value, timezone) || + !set_decoded_field(column_schema, DecodedValueKind::BOOL, max_value, + &page_statistics->max_value, timezone)) { return false; } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } return true; } -void build_page_skip_plans(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, - const std::vector& selected_ranges, int64_t row_group_rows, - std::map* page_skip_plans) { - DORIS_CHECK(page_skip_plans != nullptr); - page_skip_plans->clear(); - std::vector leaf_schemas; - collect_request_leaf_schemas(file_schema, request, &leaf_schemas); - for (const auto* leaf_schema : leaf_schemas) { - DORIS_CHECK(leaf_schema != nullptr); - ParquetPageSkipPlan page_skip_plan; - if (build_page_skip_plan_for_leaf(row_group, *leaf_schema, selected_ranges, row_group_rows, - &page_skip_plan)) { - page_skip_plans->emplace(page_skip_plan.leaf_column_id, std::move(page_skip_plan)); +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + int64_t page_rows, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + DORIS_CHECK(page_statistics != nullptr); + *page_statistics = {}; + if (!column_index.__isset.null_counts || page_idx >= column_index.null_pages.size() || + page_idx >= column_index.null_counts.size()) { + return false; + } + const int64_t null_count = column_index.null_counts[page_idx]; + const bool all_null = column_index.null_pages[page_idx]; + if (page_rows < 0 || null_count < 0 || null_count > page_rows || + all_null != (null_count == page_rows)) { + // The caller supplies the exact flat page or row-group span. Contradictory optional null + // metadata must disable pruning instead of turning a partial span into an all-null proof. + return false; + } + page_statistics->has_null_count = true; + page_statistics->has_null = null_count > 0; + page_statistics->has_not_null = !all_null; + if (!page_statistics->has_not_null) { + return true; + } + switch (column_schema.type_descriptor.physical_type) { + case tparquet::Type::BOOLEAN: + return set_native_page_boolean_min_max(column_index, column_schema, page_idx, + page_statistics, timezone); + case tparquet::Type::INT32: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case tparquet::Type::INT64: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case tparquet::Type::FLOAT: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::FLOAT, page_statistics, + timezone); + case tparquet::Type::DOUBLE: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::DOUBLE, page_statistics, + timezone); + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { + if (page_idx >= column_index.min_values.size() || + page_idx >= column_index.max_values.size()) { + return false; + } + const auto& min_value = column_index.min_values[page_idx]; + const auto& max_value = column_index.max_values[page_idx]; + const bool fixed = + column_schema.type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; + if (fixed && + (column_schema.type_descriptor.fixed_length <= 0 || + min_value.size() != static_cast(column_schema.type_descriptor.fixed_length) || + max_value.size() != static_cast(column_schema.type_descriptor.fixed_length))) { + return false; } + const auto kind = fixed ? DecodedValueKind::FIXED_BINARY : DecodedValueKind::BINARY; + if (!set_decoded_binary_field(column_schema, kind, + StringRef(min_value.data(), min_value.size()), + &page_statistics->min_value, timezone) || + !set_decoded_binary_field(column_schema, kind, + StringRef(max_value.data(), max_value.size()), + &page_statistics->max_value, timezone)) { + return false; + } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; + } + default: + return false; } } -} // namespace - -bool ParquetStatisticsUtils::TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - return build_page_statistics(column_index, column_schema, page_idx, page_statistics, timezone); +RowRange native_page_row_range(const tparquet::OffsetIndex& offset_index, size_t page_idx, + int64_t row_group_rows) { + const auto& locations = offset_index.page_locations; + const int64_t start = locations[page_idx].first_row_index; + const int64_t end = page_idx + 1 == locations.size() ? row_group_rows + : locations[page_idx + 1].first_row_index; + return {.start = start, .length = end - start}; } -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, +} // namespace + +Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, + const std::unordered_map& page_indexes, const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + const format::FileScanRequest& request, int64_t row_group_rows, std::vector* selected_ranges, std::map* page_skip_plans, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState* runtime_state) { - int64_t page_index_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &page_index_filter_time_sink + int64_t filter_time_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &filter_time_sink : &pruning_stats->page_index_filter_time); DORIS_CHECK(selected_ranges != nullptr); selected_ranges->clear(); + selected_ranges->push_back({.start = 0, .length = row_group_rows}); if (page_skip_plans != nullptr) { page_skip_plans->clear(); } - if (row_group_rows <= 0) { - return Status::OK(); - } - selected_ranges->push_back(RowRange {0, row_group_rows}); - if (!config::enable_parquet_page_index || !has_expr_zonemap_filter(request, runtime_state) || - file_reader == nullptr) { - return Status::OK(); - } - - std::shared_ptr<::parquet::PageIndexReader> page_index_reader; - std::shared_ptr<::parquet::RowGroupPageIndexReader> row_group_index_reader; - try { - if (pruning_stats != nullptr) { - ++pruning_stats->page_index_read_calls; - } - { - int64_t read_page_index_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &read_page_index_time_sink - : &pruning_stats->read_page_index_time); - page_index_reader = file_reader->GetPageIndexReader(); - if (page_index_reader == nullptr) { - return Status::OK(); - } - row_group_index_reader = page_index_reader->RowGroup(row_group_idx); - } - } catch (const ::parquet::ParquetException&) { - return Status::OK(); - } catch (const std::exception&) { + if (row_group_rows <= 0 || !config::enable_parquet_page_index || + !has_expr_zonemap_filter(request, runtime_state) || page_indexes.empty()) { return Status::OK(); } - if (row_group_index_reader == nullptr) { - return Status::OK(); + if (pruning_stats != nullptr) { + ++pruning_stats->page_index_read_calls; } std::map conjuncts_by_slot; @@ -1514,19 +1269,53 @@ Status select_row_group_ranges_by_page_index( conjuncts_by_slot[slot_index].push_back(conjunct); } } - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || + !detail::has_supported_type_defined_order(metadata, column_schema->leaf_column_id)) { + continue; + } + const auto index_it = page_indexes.find(column_schema->leaf_column_id); + if (index_it == page_indexes.end()) { + continue; + } + const auto& indexes = index_it->second; std::vector filter_ranges; - if (!select_ranges_for_expr_zonemap(row_group_index_reader, file_schema, request, - slot_index, conjuncts, row_group_rows, &filter_ranges, - pruning_stats, timezone)) { + bool usable = true; + for (size_t page_idx = 0; page_idx < indexes.offset_index.page_locations.size(); + ++page_idx) { + const auto page_range = + native_page_row_range(indexes.offset_index, page_idx, row_group_rows); + ParquetColumnStatistics statistics; + if (!build_native_page_statistics(indexes.column_index, *column_schema, page_idx, + page_range.length, &statistics, timezone)) { + usable = false; + break; + } + ZoneMapEvalContext ctx; + add_slot_zonemap(&ctx, slot_index, column_schema->type, + ParquetStatisticsUtils::MakeZoneMap(statistics)); + if (VExprContext::evaluate_zonemap_filter(conjuncts, ctx) != + ZoneMapFilterResult::kNoMatch) { + append_row_range(page_range, &filter_ranges); + } + if (pruning_stats != nullptr) { + pruning_stats->expr_zonemap_unusable_evals += ctx.stats.unusable_zonemap_eval_count; + pruning_stats->in_zonemap_point_check_count += + ctx.stats.in_zonemap_point_check_count; + pruning_stats->in_zonemap_range_only_count += ctx.stats.in_zonemap_range_only_count; + } + } + if (!usable) { continue; } *selected_ranges = intersect_ranges(*selected_ranges, filter_ranges); if (selected_ranges->empty()) { - if (page_skip_plans != nullptr) { - page_skip_plans->clear(); - } if (pruning_stats != nullptr) { pruning_stats->filtered_page_rows += row_group_rows; ++pruning_stats->filtered_row_groups_by_page_index; @@ -1534,14 +1323,37 @@ Status select_row_group_ranges_by_page_index( return Status::OK(); } } + if (page_skip_plans != nullptr) { - build_page_skip_plans(row_group_index_reader, file_schema, request, *selected_ranges, - row_group_rows, page_skip_plans); + std::vector leaves; + collect_request_leaf_schemas(file_schema, request, &leaves); + for (const auto* leaf : leaves) { + const auto index_it = page_indexes.find(leaf->leaf_column_id); + if (index_it == page_indexes.end() || leaf->max_repetition_level != 0) { + continue; + } + const auto& offset_index = index_it->second.offset_index; + ParquetPageSkipPlan skip_plan; + skip_plan.leaf_column_id = leaf->leaf_column_id; + skip_plan.skipped_pages.resize(offset_index.page_locations.size()); + skip_plan.skipped_page_compressed_sizes.resize(offset_index.page_locations.size()); + for (size_t page_idx = 0; page_idx < offset_index.page_locations.size(); ++page_idx) { + const auto range = native_page_row_range(offset_index, page_idx, row_group_rows); + if (range.length == 0 || ranges_intersect(*selected_ranges, range)) { + continue; + } + skip_plan.skipped_pages[page_idx] = 1; + skip_plan.skipped_page_compressed_sizes[page_idx] = + offset_index.page_locations[page_idx].compressed_page_size; + append_row_range(range, &skip_plan.skipped_ranges); + } + if (!skip_plan.empty()) { + page_skip_plans->emplace(skip_plan.leaf_column_id, std::move(skip_plan)); + } + } } if (pruning_stats != nullptr) { - const int64_t selected_rows = count_range_rows(*selected_ranges); - DORIS_CHECK(selected_rows <= row_group_rows); - pruning_stats->filtered_page_rows += row_group_rows - selected_rows; + pruning_stats->filtered_page_rows += row_group_rows - count_range_rows(*selected_ranges); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 9e94562bf2d37b..72381548656f9d 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include "common/status.h" @@ -27,16 +30,9 @@ #include "core/string_ref.h" #include "exprs/vexpr_fwd.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/selection_vector.h" -namespace parquet { -class BloomFilter; -class ColumnIndex; -class FileMetaData; -class ParquetFileReader; -class Statistics; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -44,6 +40,7 @@ class time_zone; namespace doris { class RuntimeState; namespace segment_v2 { +class BloomFilter; struct ZoneMap; } // namespace segment_v2 } // namespace doris @@ -51,36 +48,21 @@ struct ZoneMap; namespace doris::format::parquet { struct ParquetColumnSchema; - -// ============================================================================ -// ============================================================================ - -struct ParquetDictionaryWords { - std::vector values; - std::vector refs; - - void clear() { - values.clear(); - refs.clear(); - } - - void build_refs() { - refs.clear(); - refs.reserve(values.size()); - for (const auto& value : values) { - refs.emplace_back(value.data(), value.size()); - } - } -}; - -// Reads the PLAIN dictionary page for BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY columns and owns copied -// dictionary bytes in `values`. Both row-group pruning and row-level dictionary predicates use this -// helper so they agree on dictionary id -> Doris string value mapping. -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words); - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words); +struct ParquetFileContext; +struct ParquetTypeDescriptor; + +namespace detail { +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size); +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order); +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id); +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order); +} // namespace detail // ============================================================================ // ============================================================================ @@ -93,6 +75,7 @@ struct ParquetPruningStats { int64_t filtered_row_groups_by_bloom_filter = 0; // row groups pruned by bloom filter int64_t filtered_row_groups_by_page_index = 0; // row groups fully pruned by page index int64_t filtered_group_rows = 0; // rows in pruned row groups + int64_t filtered_bytes = 0; // requested bytes in pruned row groups int64_t filtered_page_rows = 0; // rows pruned by page index int64_t selected_row_ranges = 0; // selected row range count int64_t page_index_read_calls = 0; // Page Index read count @@ -100,6 +83,7 @@ struct ParquetPruningStats { int64_t row_group_filter_time = 0; // row-group pruning time (ns) int64_t page_index_filter_time = 0; // page-index pruning time (ns) int64_t read_page_index_time = 0; // page-index read time (ns) + int64_t parse_page_index_time = 0; // lazy page-index materialization time (ns) int64_t expr_zonemap_unusable_evals = 0; // VExpr ZoneMap unusable evaluations int64_t in_zonemap_point_check_count = 0; // VExpr IN ZoneMap point checks int64_t in_zonemap_range_only_count = 0; // VExpr IN ZoneMap range-only checks @@ -116,6 +100,20 @@ struct ParquetColumnStatistics { bool has_any_statistics() const { return has_null_count || has_min_max; } }; +struct NativeParquetPageIndex { + tparquet::ColumnIndex column_index; + tparquet::OffsetIndex offset_index; +}; + +enum class ParquetMetadataProbeMode { + ALL, + FOOTER_ONLY, + EXPENSIVE_ONLY, +}; + +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state); + // ============================================================================ // ============================================================================ // VExpr ZoneMap(TransformColumnStatistics + evaluate_zonemap_filter) @@ -128,32 +126,29 @@ struct ParquetStatisticsUtils { const ParquetColumnStatistics& statistics); static ParquetColumnStatistics TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, - const cctz::time_zone* timezone = nullptr); - - static bool TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone = nullptr); - - static bool BloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter); + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone = nullptr); + + static bool NativeBloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, + const segment_v2::BloomFilter& bloom_filter); }; Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector* candidate_row_groups, std::vector* selected_row_groups, bool enable_bloom_filter, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); + const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}, + ParquetMetadataProbeMode probe_mode = ParquetMetadataProbeMode::ALL); -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, +Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, + const std::unordered_map& page_indexes, const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, + const format::FileScanRequest& request, int64_t row_group_rows, std::vector* selected_ranges, std::map* page_skip_plans, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, const RuntimeState* runtime_state = nullptr); diff --git a/be/src/format_v2/parquet/parquet_type.cpp b/be/src/format_v2/parquet/parquet_type.cpp index d35181d0397178..be5ecd1c8b0772 100644 --- a/be/src/format_v2/parquet/parquet_type.cpp +++ b/be/src/format_v2/parquet/parquet_type.cpp @@ -17,339 +17,31 @@ #include "format_v2/parquet/parquet_type.h" -#include - -#include -#include - -#include "core/data_type/data_type_factory.hpp" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/primitive_type.h" - namespace doris::format::parquet { -namespace { - -DataTypePtr create_type(PrimitiveType type, bool nullable, int precision = 0, int scale = 0) { - return DataTypeFactory::instance().create_data_type(type, nullable, precision, scale); -} - -PrimitiveType decimal_primitive_type(int precision) { - return precision > 38 ? TYPE_DECIMAL256 : TYPE_DECIMAL128I; -} - -void mark_decimal(const ::parquet::ColumnDescriptor* column, int precision, int scale, - ParquetTypeDescriptor* result) { - result->is_decimal = true; - result->decimal_precision = precision; - result->decimal_scale = scale; - switch (column->physical_type()) { - case ::parquet::Type::INT32: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; - break; - case ::parquet::Type::INT64: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; - break; - default: - result->extra_type_info = ParquetExtraTypeInfo::NONE; - break; - } -} - -void mark_integer(int bit_width, bool is_signed, ParquetTypeDescriptor* result) { - result->integer_bit_width = bit_width; - result->is_unsigned_integer = !is_signed; -} - -DataTypePtr converted_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const bool nullable = column->max_definition_level() > 0; - switch (column->converted_type()) { - case ::parquet::ConvertedType::UTF8: - case ::parquet::ConvertedType::ENUM: - case ::parquet::ConvertedType::JSON: - case ::parquet::ConvertedType::BSON: - return create_type(TYPE_STRING, nullable); - case ::parquet::ConvertedType::DECIMAL: - mark_decimal(column, column->type_precision(), column->type_scale(), result); - return create_type(decimal_primitive_type(column->type_precision()), nullable, - column->type_precision(), column->type_scale()); - case ::parquet::ConvertedType::DATE: - return create_type(TYPE_DATEV2, nullable); - case ::parquet::ConvertedType::TIME_MILLIS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIME_MICROS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIMESTAMP_MILLIS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 3); - case ::parquet::ConvertedType::TIMESTAMP_MICROS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 6); - // Parquet stores signed and unsigned integer logical annotations on signed physical carriers: - // INT_8/UINT_8/INT_16/UINT_16/INT_32/UINT_32 use physical INT32, and - // INT_64/UINT_64 use physical INT64. Doris maps unsigned integers to the next wider - // signed type so all values in the unsigned range can be represented. - case ::parquet::ConvertedType::INT_8: - mark_integer(8, true, result); - return create_type(TYPE_TINYINT, nullable); - case ::parquet::ConvertedType::UINT_8: - mark_integer(8, false, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::INT_16: - mark_integer(16, true, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::UINT_16: - mark_integer(16, false, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::INT_32: - mark_integer(32, true, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::UINT_32: - mark_integer(32, false, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::INT_64: - mark_integer(64, true, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::UINT_64: - mark_integer(64, false, result); - return create_type(TYPE_LARGEINT, nullable); - case ::parquet::ConvertedType::NONE: - default: - return nullptr; - } -} - -DataTypePtr logical_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const auto& logical_type = column->logical_type(); - if (logical_type == nullptr || !logical_type->is_valid() || logical_type->is_none()) { - return nullptr; - } - const bool nullable = column->max_definition_level() > 0; - if (logical_type->is_string() || logical_type->is_enum() || logical_type->is_JSON() || - logical_type->is_BSON() || logical_type->is_UUID()) { - return create_type(TYPE_STRING, nullable); - } - if (logical_type->is_decimal()) { - const auto& decimal_type = static_cast(*logical_type); - mark_decimal(column, decimal_type.precision(), decimal_type.scale(), result); - return create_type(decimal_primitive_type(decimal_type.precision()), nullable, - decimal_type.precision(), decimal_type.scale()); - } - if (logical_type->is_date()) { - return create_type(TYPE_DATEV2, nullable); - } - if (logical_type->is_time()) { - const auto& time_type = static_cast(*logical_type); - if (time_type.is_adjusted_to_utc()) { - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - } - int scale = 0; - if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else { - return nullptr; - } - return create_type(TYPE_TIMEV2, nullable, 0, scale); - } - if (logical_type->is_timestamp()) { - const auto& timestamp_type = - static_cast(*logical_type); - int scale = 0; - if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::NANOS) { - scale = 6; - result->time_unit = ParquetTimeUnit::NANOS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_NS; - } else { - return nullptr; - } - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = timestamp_type.is_adjusted_to_utc(); - return create_type(TYPE_DATETIMEV2, nullable, 0, scale); - } - if (logical_type->is_int()) { - const auto& int_type = static_cast(*logical_type); - mark_integer(int_type.bit_width(), int_type.is_signed(), result); - switch (int_type.bit_width()) { - case 8: - return create_type(int_type.is_signed() ? TYPE_TINYINT : TYPE_SMALLINT, nullable); - case 16: - return create_type(int_type.is_signed() ? TYPE_SMALLINT : TYPE_INT, nullable); - case 32: - return create_type(int_type.is_signed() ? TYPE_INT : TYPE_BIGINT, nullable); - case 64: - return create_type(int_type.is_signed() ? TYPE_BIGINT : TYPE_LARGEINT, nullable); - default: - return nullptr; - } - } - if (logical_type->is_float16()) { - if (column->physical_type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY || - column->type_length() != 2) { - return nullptr; - } - result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; - return create_type(TYPE_FLOAT, nullable); - } - return nullptr; -} - -DataTypePtr physical_type_to_doris_type(const ::parquet::ColumnDescriptor* column) { - const bool nullable = column->max_definition_level() > 0; - DataTypePtr type; - switch (column->physical_type()) { - case ::parquet::Type::BOOLEAN: - type = std::make_shared(); - break; - case ::parquet::Type::INT32: - type = std::make_shared(); - break; - case ::parquet::Type::INT64: - type = std::make_shared(); - break; - case ::parquet::Type::FLOAT: - type = std::make_shared(); - break; - case ::parquet::Type::DOUBLE: - type = std::make_shared(); - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - type = std::make_shared(); - break; - case ::parquet::Type::INT96: - type = create_type(TYPE_DATETIMEV2, nullable, 0, 6); - break; - default: - return nullptr; - } - return nullable ? make_nullable(type) : type; -} - -bool record_reader_physical_type_supported(::parquet::Type::type physical_type) { - switch (physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::INT96: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return true; - default: - return false; - } -} - -} // namespace - -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column) { - if (column == nullptr) { - return {}; - } - auto path = column->path(); - if (path) { - return path->ToDotString(); - } - return column->name(); -} - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column) { - ParquetTypeDescriptor result; - if (column == nullptr) { - return result; - } - - result.physical_type = column->physical_type(); - result.converted_type = column->converted_type(); - result.fixed_length = column->type_length(); - - if (auto logical_type = logical_type_to_doris_type(column, &result); logical_type != nullptr) { - result.doris_type = logical_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - result.supports_record_reader = false; - } else if (auto converted_type = converted_type_to_doris_type(column, &result); - converted_type != nullptr) { - result.doris_type = converted_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - result.supports_record_reader = false; - } else { - result.doris_type = physical_type_to_doris_type(column); - if (result.physical_type == ::parquet::Type::INT96) { - result.extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; - } - } - - result.is_string_like = !result.is_decimal && - result.extra_type_info != ParquetExtraTypeInfo::FLOAT16 && - (result.physical_type == ::parquet::Type::BYTE_ARRAY || - result.physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - - if (!record_reader_physical_type_supported(result.physical_type)) { - result.supports_record_reader = false; - } - return result; -} - -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor) { - return type_descriptor.supports_record_reader; -} DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor) { switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: + case tparquet::Type::BOOLEAN: return DecodedValueKind::BOOL; - case ::parquet::Type::INT32: + case tparquet::Type::INT32: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 32) { return DecodedValueKind::UINT32; } return DecodedValueKind::INT32; - case ::parquet::Type::INT64: + case tparquet::Type::INT64: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 64) { return DecodedValueKind::UINT64; } return DecodedValueKind::INT64; - case ::parquet::Type::INT96: + case tparquet::Type::INT96: return DecodedValueKind::INT96; - case ::parquet::Type::FLOAT: + case tparquet::Type::FLOAT: return DecodedValueKind::FLOAT; - case ::parquet::Type::DOUBLE: + case tparquet::Type::DOUBLE: return DecodedValueKind::DOUBLE; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return DecodedValueKind::FIXED_BINARY; - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BYTE_ARRAY: default: return DecodedValueKind::BINARY; } diff --git a/be/src/format_v2/parquet/parquet_type.h b/be/src/format_v2/parquet/parquet_type.h index 5d21aae6bae092..7ab3d2b3b39d8d 100644 --- a/be/src/format_v2/parquet/parquet_type.h +++ b/be/src/format_v2/parquet/parquet_type.h @@ -15,17 +15,13 @@ #pragma once -#include +#include #include #include "core/data_type/data_type.h" #include "core/data_type_serde/decoded_column_view.h" -namespace parquet { -class ColumnDescriptor; -} // namespace parquet - namespace doris::format::parquet { // ============================================================================ @@ -53,11 +49,15 @@ enum class ParquetTimeUnit { // ============================================================================ // ============================================================================ struct ParquetTypeDescriptor { + // Keep the V2 semantic tree on generated Thrift enums; using parquet-cpp descriptors here + // would reintroduce a second metadata model and make native planning bypassable. DataTypePtr doris_type; + // Physical fallback used only to keep file schema construction alive when the logical type is + // unsupported. Column reader creation still rejects unsupported_reason before decoding. + DataTypePtr physical_doris_type; ParquetExtraTypeInfo extra_type_info = ParquetExtraTypeInfo::NONE; ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; - ::parquet::Type::type physical_type = ::parquet::Type::UNDEFINED; - ::parquet::ConvertedType::type converted_type = ::parquet::ConvertedType::UNDEFINED; + tparquet::Type::type physical_type = tparquet::Type::INT32; int integer_bit_width = -1; // bit width for INT_8/16/32/64 int decimal_precision = -1; // precision for DECIMAL(p,s) int decimal_scale = -1; // scale for DECIMAL(p,s) @@ -67,16 +67,9 @@ struct ParquetTypeDescriptor { bool is_timestamp = false; // whether this is a timestamp type bool timestamp_is_adjusted_to_utc = false; // whether the timestamp is UTC-normalized bool is_string_like = false; // binary type that is neither decimal nor FLOAT16 - bool supports_record_reader = true; // whether Arrow RecordReader can read this type std::string unsupported_reason; // non-empty when this Parquet logical type is unsupported }; -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column); - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column); - -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor); - DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor); } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.cpp b/be/src/format_v2/parquet/reader/column_reader.cpp index 352fbbd7c3d215..db5def75b8c8a7 100644 --- a/be/src/format_v2/parquet/reader/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/column_reader.cpp @@ -15,145 +15,25 @@ #include "format_v2/parquet/reader/column_reader.h" -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include #include -#include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "runtime/runtime_profile.h" namespace doris::format::parquet { -namespace { - -class DataPageSkipFilter { -public: - DataPageSkipFilter(const ParquetPageSkipPlan* page_skip_plan, - ParquetPageSkipProfile page_skip_profile) - : _page_skip_plan(page_skip_plan), _page_skip_profile(page_skip_profile) { - DORIS_CHECK(_page_skip_plan != nullptr); - } - - bool operator()(const ::parquet::DataPageStats&) { - // Arrow invokes this callback once for each DATA_PAGE/DATA_PAGE_V2 and never for - // dictionary pages, so this ordinal matches Parquet OffsetIndex page locations. - const size_t page_idx = _next_data_page_idx++; - const bool skip = _page_skip_plan->should_skip_page(page_idx); - if (!skip) { - return false; - } - update_skip_profile(page_idx); - return true; - } - -private: - void update_skip_profile(size_t page_idx) const { - if (_page_skip_profile.skipped_pages != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); - } - if (_page_skip_profile.skipped_bytes != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_bytes, - _page_skip_plan->skipped_page_compressed_size(page_idx)); - } - } - - const ParquetPageSkipPlan* _page_skip_plan = nullptr; - ParquetPageSkipProfile _page_skip_profile; - size_t _next_data_page_idx = 0; -}; - -const ParquetPageSkipPlan* find_page_skip_plan( - const std::map* page_skip_plans, int leaf_column_id) { - if (page_skip_plans == nullptr) { - return nullptr; - } - const auto plan_it = page_skip_plans->find(leaf_column_id); - return plan_it == page_skip_plans->end() ? nullptr : &plan_it->second; -} - -void install_data_page_filter(std::unique_ptr<::parquet::PageReader>& page_reader, - const std::map* page_skip_plans, - int leaf_column_id, ParquetPageSkipProfile page_skip_profile) { - DORIS_CHECK(page_reader != nullptr); - const ParquetPageSkipPlan* page_skip_plan = - find_page_skip_plan(page_skip_plans, leaf_column_id); - if (page_skip_plan == nullptr) { - return; - } - page_reader->set_data_page_filter(DataPageSkipFilter(page_skip_plan, page_skip_profile)); -} - -bool supports_nested_scalar_record_reader(const ParquetColumnSchema& column_schema) { - if (column_schema.type_descriptor.supports_record_reader) { - return true; - } - const auto& type_descriptor = column_schema.type_descriptor; - if ((type_descriptor.extra_type_info != ParquetExtraTypeInfo::NONE && - type_descriptor.extra_type_info != ParquetExtraTypeInfo::FLOAT16) || - type_descriptor.is_decimal || type_descriptor.is_timestamp || - type_descriptor.is_string_like) { - return false; - } - if (type_descriptor.converted_type != ::parquet::ConvertedType::NONE && - type_descriptor.converted_type != ::parquet::ConvertedType::UNDEFINED) { - return false; - } - switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - return true; - default: - return false; - } - return true; -} -} // namespace +ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, + ParquetColumnReaderProfile profile) + : _profile(profile), + _field_id(schema.local_id), + _leaf_column_id(schema.leaf_column_id), + _type(std::move(type)), + _name(schema.name) {} Status ParquetColumnReader::skip(int64_t rows) { return Status::NotSupported("Parquet column skip is not implemented, rows={}", rows); } -void ParquetColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - int64_t child_cursor = nested_build_level_cursor(); - const auto& child_rep_levels = nested_repetition_levels(); - const int64_t child_levels_written = nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!is_or_has_repeated_child() || child_rep_level <= parent_repetition_level) { - break; - } - } - set_nested_build_level_cursor(child_cursor); -} - void ParquetColumnReader::update_reader_read_rows(int64_t rows) const { if (_profile.reader_read_rows != nullptr) { COUNTER_UPDATE(_profile.reader_read_rows, rows); @@ -166,21 +46,16 @@ void ParquetColumnReader::update_reader_skip_rows(int64_t rows) const { } } -Status ParquetColumnReader::select(const SelectionVector& sel, uint16_t selected_rows, +Status ParquetColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, MutableColumnPtr& column) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Parquet selected read result is null for column {}", - name()); - } - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); + DORIS_CHECK(column); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); - const auto ranges = selection_to_ranges(sel, selected_rows); + const auto ranges = selection_to_ranges(selection, selected_rows); int64_t cursor = 0; for (const auto& range : ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument("Invalid parquet selection range [{}, {}) for column {}", - range.start, range.start + range.length, name()); - } + DORIS_CHECK(range.start >= cursor); + DORIS_CHECK(range.start + range.length <= batch_rows); RETURN_IF_ERROR(skip(range.start - cursor)); int64_t range_rows_read = 0; @@ -206,453 +81,15 @@ Status ParquetColumnReader::select_with_dictionary_filter(const SelectionVector& name()); } -ParquetColumnReaderFactory::ParquetColumnReaderFactory( - std::shared_ptr<::parquet::RowGroupReader> row_group, int num_leaf_columns, - const std::map* page_skip_plans, - ParquetPageSkipProfile page_skip_profile, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile column_reader_profile) - : _row_group(std::move(row_group)), - _record_readers(static_cast(num_leaf_columns)), - _dictionary_record_readers(static_cast(num_leaf_columns)), - _page_skip_plans(page_skip_plans), - _page_skip_profile(page_skip_profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _column_reader_profile(column_reader_profile) {} - -std::unique_ptr ParquetColumnReaderFactory::create_row_position_column_reader( - int64_t row_group_first_row) const { - return std::make_unique(row_group_first_row, _column_reader_profile); -} - -std::unique_ptr ParquetColumnReaderFactory::create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const { - return std::make_unique(context, row_group_first_row, - _column_reader_profile); -} - -Status ParquetColumnReaderFactory::make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, bool use_page_skip_plan, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - const auto* page_skip_plan = - use_page_skip_plan ? find_page_skip_plan(_page_skip_plans, column_schema.leaf_column_id) - : nullptr; - *reader = std::make_unique(column_schema, std::move(record_reader), - page_skip_plan, _timezone, _enable_strict_mode, - _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_scalar_column_reader( - const ParquetColumnSchema& column_schema, bool is_nested, bool read_dictionary, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (!column_schema.type_descriptor.unsupported_reason.empty()) { - return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, - column_schema.type_descriptor.unsupported_reason); - } - if (is_nested && column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { - return Status::InvalidArgument("Parquet nested scalar reader requires primitive column {}", - column_schema.name); - } - if (column_schema.leaf_column_id < 0 || - column_schema.leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - column_schema.leaf_column_id, column_schema.name); - } - if (column_schema.descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", - column_schema.name); - } - if (!is_nested && (column_schema.descriptor->max_repetition_level() != 0 || - column_schema.descriptor->max_definition_level() > 1)) { - return Status::NotSupported( - "Current parquet scalar reader only supports flat primitive columns; column {} is " - "not supported", - column_schema.name); - } - if (is_nested && !supports_nested_scalar_record_reader(column_schema)) { - return Status::NotSupported( - "Current parquet nested scalar reader does not support column {}", - column_schema.name); - } - if (!is_nested && !column_schema.type_descriptor.supports_record_reader) { - return Status::NotSupported("Current parquet scalar reader does not support column {}", - column_schema.name); - } - std::shared_ptr<::parquet::internal::RecordReader> record_reader; - // Nested readers implement skip() by materializing rows into a scratch column. If Arrow - // page filtering is also installed, those scratch reads can consume the next selected row - // after a page-index range gap. Keep page filtering on flat scalar readers only. - RETURN_IF_ERROR(get_record_reader(column_schema.leaf_column_id, column_schema.descriptor, - column_schema.name, !is_nested, read_dictionary, - &record_reader)); - return make_scalar_column_reader(column_schema, std::move(record_reader), !is_nested, reader); -} - -// 1. RowGroupReader::GetColumnPageReader(leaf_column_id) -> Arrow PageReader -Status ParquetColumnReaderFactory::get_record_reader( - int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, const std::string& name, - bool install_page_filter, bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (_row_group == nullptr) { - return Status::InternalError("Parquet row group reader is not initialized for column {}", - name); - } - if (leaf_column_id < 0 || leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - leaf_column_id, name); - } - if (descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", name); - } - auto& record_readers = read_dictionary ? _dictionary_record_readers : _record_readers; - if (record_readers[leaf_column_id] == nullptr) { - try { - auto page_reader = _row_group->GetColumnPageReader(leaf_column_id); - if (install_page_filter) { - install_data_page_filter(page_reader, _page_skip_plans, leaf_column_id, - _page_skip_profile); - } - const auto level_info = ::parquet::internal::LevelInfo::ComputeLevelInfo(descriptor); - record_readers[leaf_column_id] = ::parquet::internal::RecordReader::Make( - descriptor, level_info, ::arrow::default_memory_pool(), - /*read_dictionary=*/read_dictionary, - /*read_dense_for_nullable=*/false); - record_readers[leaf_column_id]->SetPageReader(std::move(page_reader)); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } - } - if (record_readers[leaf_column_id] == nullptr) { - return Status::Corruption("Failed to create parquet record reader for column {}", name); - } - *reader = record_readers[leaf_column_id]; - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_struct_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - std::vector> child_readers; - child_readers.reserve(column_schema.children.size()); - std::vector child_output_indices; - child_output_indices.reserve(column_schema.children.size()); - DataTypes projected_child_types; - Strings projected_child_names; - for (size_t child_idx = 0; child_idx < column_schema.children.size(); ++child_idx) { - const auto& child_schema = column_schema.children[child_idx]; - const auto* child_projection = - format::find_child_projection(projection, child_schema->local_id); - if (!format::is_child_projected(projection, child_schema->local_id)) { - continue; - } - std::unique_ptr child_reader; - RETURN_IF_ERROR( - create_column_reader(*child_schema, child_projection, true, false, &child_reader)); - child_output_indices.push_back(static_cast(projected_child_types.size())); - projected_child_types.push_back(make_nullable(child_reader->type())); - projected_child_names.push_back(child_reader->name()); - child_readers.push_back(std::move(child_reader)); - } - if (format::is_partial_projection(projection) && - projected_child_types.size() != projection->children.size()) { - return Status::InvalidArgument( - "Parquet STRUCT projection for column {} contains invalid child", - column_schema.name); - } - if (projected_child_types.empty() && !column_schema.children.empty()) { - return Status::NotSupported("Parquet STRUCT projection for column {} contains no children", - column_schema.name); - } - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(projection)) { - type = std::make_shared(projected_child_types, projected_child_names); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique( - column_schema, std::move(type), std::move(child_readers), - std::move(child_output_indices), _column_reader_profile); +Status ParquetColumnReader::select_with_fixed_width_filter(const SelectionVector&, uint16_t, + int64_t, const VExprSPtrs&, int, + IColumn*, IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *used_filter = false; return Status::OK(); } -Status ParquetColumnReaderFactory::create_list_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for column {}", - column_schema.name); - } - std::unique_ptr element_reader; - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - if (format::is_partial_projection(projection) && element_projection == nullptr) { - return Status::NotSupported("Parquet LIST projection for column {} contains no element", - column_schema.name); - } - RETURN_IF_ERROR( - create_column_reader(element_schema, element_projection, true, false, &element_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(element_projection)) { - type = std::make_shared(element_reader->type()); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique(column_schema, std::move(type), - std::move(element_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_map_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP layout for column {}", - column_schema.name); - } - const auto& key_schema = *column_schema.children[0]; - const auto& value_schema = *column_schema.children[1]; - const auto* value_projection = format::find_child_projection(projection, value_schema.local_id); - if (format::is_partial_projection(projection)) { - if (value_projection == nullptr) { - return Status::NotSupported("Parquet MAP projection for column {} contains no value", - column_schema.name); - } - for (const auto& child_projection : projection->children) { - if (child_projection.local_id() == key_schema.local_id) { - continue; - } - if (child_projection.local_id() != value_schema.local_id) { - return Status::InvalidArgument( - "Parquet MAP projection for column {} contains invalid child", - column_schema.name); - } - } - } - std::unique_ptr key_reader; - // MAP materialization always needs the full key stream. It owns entry existence, offsets and - // key equality semantics, so MAP projection is defined only as value-subtree pruning. - RETURN_IF_ERROR(create_column_reader(key_schema, nullptr, true, false, &key_reader)); - std::unique_ptr value_reader; - RETURN_IF_ERROR( - create_column_reader(value_schema, value_projection, true, false, &value_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(value_projection)) { - type = std::make_shared(make_nullable(key_reader->type()), - make_nullable(value_reader->type())); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = - std::make_unique(column_schema, std::move(type), std::move(key_reader), - std::move(value_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, - bool read_dictionary) const { - return create_column_reader(column_schema, projection, false, read_dictionary, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - return create_count_shape_reader_impl(column_schema, projection, false, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader_impl( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, is_nested, false, reader); - case ParquetColumnSchemaKind::STRUCT: { - if (column_schema.children.empty()) { - return Status::NotSupported("Parquet COUNT shape reader found empty STRUCT column {}", - column_schema.name); - } - const ParquetColumnSchema* child_schema = nullptr; - const format::LocalColumnIndex* child_projection = nullptr; - if (format::is_partial_projection(projection)) { - const auto child_id = projection->children[0].local_id(); - const auto child_it = std::ranges::find_if( - column_schema.children, - [&](const auto& child) { return child->local_id == child_id; }); - if (child_it == column_schema.children.end()) { - return Status::InvalidArgument( - "Parquet COUNT projection for column {} contains invalid child", - column_schema.name); - } - child_schema = child_it->get(); - child_projection = &projection->children[0]; - } else { - child_schema = column_schema.children[0].get(); - } - DORIS_CHECK(child_schema != nullptr); - return create_count_shape_reader_impl(*child_schema, child_projection, true, reader); - } - case ParquetColumnSchemaKind::LIST: { - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for COUNT column {}", - column_schema.name); - } - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - return create_count_shape_reader_impl(element_schema, element_projection, true, reader); - } - case ParquetColumnSchemaKind::MAP: { - if (column_schema.children.empty()) { - return Status::NotSupported("Unsupported parquet MAP layout for COUNT column {}", - column_schema.name); - } - // The key stream defines MAP entry existence and offsets. Counting top-level MAP NULL-ness - // from it avoids creating a value reader, which is the expensive path for files with huge - // MAP value strings. - return create_count_shape_reader_impl(*column_schema.children[0], nullptr, true, reader); - } - } - return Status::NotSupported("Unsupported parquet column schema kind for COUNT column {}", - column_schema.name); -} - -Status ParquetColumnReaderFactory::create_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, bool read_dictionary, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (is_nested) { - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet scalar projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, true, false, reader); - } - return create_scalar_column_reader(column_schema, false, read_dictionary, reader); - case ParquetColumnSchemaKind::STRUCT: - return create_struct_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::LIST: - return create_list_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::MAP: - return create_map_column_reader(column_schema, projection, reader); - } - return Status::NotSupported("Unsupported parquet column schema kind for column {}", - column_schema.name); -} - -ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, - ParquetColumnReaderProfile profile) - : _profile(profile), - _field_id(schema.local_id), - _leaf_column_id(schema.leaf_column_id), - _nullable_definition_level(schema.nullable_definition_level), - _repeated_repetition_level(schema.repeated_repetition_level), - _definition_level(schema.definition_level), - _repetition_level(schema.repetition_level), - _repeated_ancestor_definition_level(schema.repeated_ancestor_definition_level), - _type(std::move(type)), - _name(schema.name) {} - -Status ParquetColumnReader::load_nested_batch(int64_t) { - return Status::NotSupported("Parquet nested batch load is not supported for column {}", _name); -} - -Status ParquetColumnReader::load_nested_levels_batch(int64_t) { - return Status::NotSupported("Parquet nested levels batch load is not supported for column {}", - _name); -} - -Status ParquetColumnReader::build_nested_column(int64_t, MutableColumnPtr&, int64_t*) { - return Status::NotSupported("Parquet nested column build is not supported for column {}", - _name); -} - -Status ParquetColumnReader::consume_nested_column(int64_t, int64_t*) { - return Status::NotSupported("Parquet nested column consume is not supported for column {}", - _name); -} - -Status ParquetColumnReader::skip_nested_rows(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - // A nested parent row may expand to many child values. Capping the number of parent rows per - // loaded batch bounds that amplification for large holes. The consume interface advances the - // loaded definition/repetition levels recursively without constructing a discarded Column. - constexpr int64_t MAX_NESTED_SKIP_BATCH_SIZE = 4096; - int64_t remaining_rows = rows; - while (remaining_rows > 0) { - const int64_t batch_rows = std::min(remaining_rows, MAX_NESTED_SKIP_BATCH_SIZE); - RETURN_IF_ERROR(load_nested_levels_batch(batch_rows)); - int64_t rows_consumed = 0; - RETURN_IF_ERROR(consume_nested_column(batch_rows, &rows_consumed)); - if (rows_consumed != batch_rows) { - return Status::Corruption( - "Failed to skip nested parquet column {}: skipped {} of {} rows in batch", - _name, rows_consumed, batch_rows); - } - remaining_rows -= batch_rows; - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -const std::vector& ParquetColumnReader::nested_definition_levels() const { - static const std::vector empty; - return empty; -} - -const std::vector& ParquetColumnReader::nested_repetition_levels() const { - static const std::vector empty; - return empty; -} - -int64_t ParquetColumnReader::nested_levels_written() const { - return 0; -} - -bool ParquetColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index 51dbd44c11c226..6914a5be7d165d 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -16,216 +16,81 @@ #pragma once #include -#include -#include #include -#include #include "common/status.h" -#include "core/column/column_nullable.h" +#include "core/column/column.h" #include "core/data_type/data_type.h" -#include "format_v2/column_data.h" +#include "exprs/vexpr_fwd.h" #include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" #include "format_v2/parquet/selection_vector.h" -#include "runtime/runtime_profile.h" - -namespace parquet { -class ColumnDescriptor; -class RowGroupReader; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris { -class IColumn; -} // namespace doris namespace doris::format::parquet { struct ParquetColumnSchema; +// Scan-time column contract for FileScannerV2. +// +// Physical Parquet columns are implemented by NativeColumnReader, which owns Doris' native page +// decoder and writes directly into the destination Doris column. Synthetic row-position/global-id +// readers implement the same cursor contract. Arrow RecordReader, ParquetLeafBatch, decoded value +// views, and the former nested build/consume protocol intentionally do not belong to this API. class ParquetColumnReader { public: virtual ~ParquetColumnReader() = default; virtual int file_column_id() const { return _field_id; } - virtual int parquet_leaf_column_id() const { return _leaf_column_id; } - - int16_t nullable_definition_level() const { return _nullable_definition_level; } - int16_t repeated_repetition_level() const { return _repeated_repetition_level; } - virtual const DataTypePtr& type() const { return _type; } virtual const std::string& name() const { return _name; } const ParquetColumnReaderProfile& profile() const { return _profile; } + // Consume rows from the row-group cursor and append them to column. virtual Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) = 0; + // Consume rows without materializing values. virtual Status skip(int64_t rows); - virtual Status select(const SelectionVector& sel, uint16_t selected_rows, int64_t batch_rows, - MutableColumnPtr& column); + // Consume batch_rows and append only selection[0, selected_rows). The default implementation + // coalesces adjacent indices into ranges; native readers override it with one decoder call. + virtual Status select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column); - virtual Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, + virtual Status select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter); - virtual Status load_nested_batch(int64_t rows); - - // Shape-only load interface for COUNT(col) and skip. Implementations guarantee only that - // nested_definition_levels(), nested_repetition_levels(), and nested_levels_written() are - // available; value indices and payload columns may be absent. Callers may inspect the levels or - // call consume_nested_column(), but must not call build_nested_column() afterwards. For example, - // skipping ARRAY uses this method to find ARRAY boundaries without constructing a - // ColumnString. The underlying Arrow reader may still decode a page because it has no public - // levels-only API. Normal scans that need output values use load_nested_batch() instead. - virtual Status load_nested_levels_batch(int64_t rows); - - virtual Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read); - - // Consume logical values from a batch previously loaded by load_nested_batch() or - // load_nested_levels_batch() without appending them to an output Column. Implementations must - // advance exactly the same nested level cursors and perform the same shape/null/alignment - // validation as build_nested_column(). The levels-only form is preferred for skip paths because - // it avoids transferring leaf payloads into Doris Columns when they will be discarded. - // - // `length_upper_bound` is expressed at this reader's logical level, not in physical leaf - // values. For example, consuming two rows from ARRAY [[1, 2], []] consumes two parent ARRAY - // rows but only two element values. A MAP implementation must also consume key/value streams - // in lockstep, while a nullable STRUCT consumes no child value for a null parent. - // - // Callers must not use the ordinary skip() after either load call: the leaf stream has already - // advanced into an in-memory nested batch, and doing so would advance it twice. - // `values_consumed` may be smaller than the requested bound only when the loaded batch ends. - virtual Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed); - - virtual const std::vector& nested_definition_levels() const; - virtual const std::vector& nested_repetition_levels() const; - virtual int64_t nested_levels_written() const; - virtual bool is_or_has_repeated_child() const; - virtual void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level); - - int64_t nested_build_level_cursor() const { return _nested_build_level_cursor; } - void set_nested_build_level_cursor(int64_t cursor) { - DORIS_CHECK(cursor >= 0); - _nested_build_level_cursor = cursor; + // Consume batch_rows and evaluate eligible fixed-width values without first constructing a + // complete predicate column. Append survivors when projected_column is non-null. Implementations + // must leave the cursor untouched when used_filter=false. + virtual Status select_with_fixed_width_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const VExprSPtrs& conjuncts, int column_id, + IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter); + + // Native statistics are cumulative and can be recursively aggregated for complex columns. + // Flush once at the scheduler batch boundary instead of snapshotting after each operation. + virtual void flush_profile() {} + virtual bool crossed_page_since_last_batch() { return false; } + virtual Result dictionary_values() { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); } - void reset_nested_build_level_cursor() { _nested_build_level_cursor = 0; } protected: - ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, + ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, ParquetColumnReaderProfile profile = {}); ParquetColumnReader() = default; - // Load shape levels and consume skipped parent rows in bounded batches. The bound limits level - // memory when a parent expands to many children; the levels-only load plus - // consume_nested_column() avoids payload materialization and output Columns. - Status skip_nested_rows(int64_t rows); + void update_reader_read_rows(int64_t rows) const; void update_reader_skip_rows(int64_t rows) const; ParquetColumnReaderProfile _profile; - const int _field_id = -1; // child ordinal in the parent node - const int _leaf_column_id = -1; // Parquet physical leaf column id (-1 = non-leaf) - const int16_t _nullable_definition_level = - 0; // definition-level threshold where this node becomes nullable - const int16_t _repeated_repetition_level = - 0; // repetition level of the nearest repeated ancestor - const int16_t _definition_level = 0; // definition level accumulated to this node - const int16_t _repetition_level = 0; // repetition level accumulated to this node - const int16_t _repeated_ancestor_definition_level = - 0; // definition level of the nearest repeated ancestor - const DataTypePtr _type; // Doris target type - const std::string _name; // column name for error messages - int64_t _nested_build_level_cursor = 0; // nested build cursor (current level position) + const int _field_id = -1; + const int _leaf_column_id = -1; + const DataTypePtr _type; + const std::string _name; }; -class ParquetColumnReaderFactory { -public: - ParquetColumnReaderFactory(std::shared_ptr<::parquet::RowGroupReader> row_group, - int num_leaf_columns, - const std::map* page_skip_plans = nullptr, - ParquetPageSkipProfile page_skip_profile = {}, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false, - ParquetColumnReaderProfile column_reader_profile = {}); - - Status create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, bool read_dictionary = false) const; - - // Create a scalar reader for one representative leaf that carries the top-level column shape. - // This is used by COUNT(col): the caller needs definition/repetition levels to decide whether - // the top-level value is NULL, but must not materialize heavy payload leaves. MAP deliberately - // uses the key leaf because the key stream owns entry existence and avoids reading value pages. - Status create_count_shape_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create(const ParquetColumnSchema& column_schema, - std::unique_ptr* reader) const { - return create(column_schema, nullptr, reader); - } - - std::unique_ptr create_row_position_column_reader( - int64_t row_group_first_row) const; - std::unique_ptr create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const; - -private: - Status create_scalar_column_reader(const ParquetColumnSchema& column_schema, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - - Status create_struct_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_list_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_map_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - Status create_count_shape_reader_impl(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - bool is_nested, - std::unique_ptr* reader) const; - - Status get_record_reader(int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, - const std::string& name, bool install_page_filter, - bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const; - - Status make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - bool use_page_skip_plan, std::unique_ptr* reader) const; - - std::shared_ptr<::parquet::RowGroupReader> _row_group; // Arrow RowGroup reader - mutable std::vector> - _record_readers; // RecordReader cache by leaf_column_id - mutable std::vector> - _dictionary_record_readers; // dictionary-exposing RecordReader cache by leaf_column_id - const std::map* _page_skip_plans = - nullptr; // page-index pruning result - ParquetPageSkipProfile _page_skip_profile; // page skip profile - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - ParquetColumnReaderProfile _column_reader_profile; // column reader profile -}; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp new file mode 100644 index 00000000000000..171107107f1a38 --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/count_column_reader.h" + +#include +#include +#include + +#include "common/config.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet { +namespace { + +Status find_count_leaf(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const ParquetColumnSchema** leaf) { + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + if (format::is_partial_projection(projection)) { + return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", + schema.name); + } + *leaf = &schema; + return Status::OK(); + case ParquetColumnSchemaKind::STRUCT: { + DORIS_CHECK(!schema.children.empty()); + if (!format::is_partial_projection(projection)) { + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + DORIS_CHECK(!projection->children.empty()); + const auto child_id = projection->children.front().local_id(); + const auto child = std::ranges::find_if(schema.children, [child_id](const auto& candidate) { + return candidate->local_id == child_id; + }); + if (child == schema.children.end()) { + return Status::InvalidArgument( + "Parquet COUNT projection for column {} contains invalid child", schema.name); + } + return find_count_leaf(**child, &projection->children.front(), leaf); + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto& element = *schema.children.front(); + return find_count_leaf(element, format::find_child_projection(projection, element.local_id), + leaf); + } + case ParquetColumnSchemaKind::MAP: + // The key leaf owns entry existence and top-level MAP shape. Choosing it also avoids + // reading a potentially huge value BYTE_ARRAY for COUNT(map_col). + DORIS_CHECK(!schema.children.empty()); + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + return Status::InternalError("Unknown Parquet schema kind for column {}", schema.name); +} + +NativeFieldSchema* find_physical_leaf(NativeFieldSchema* field, int physical_column_index) { + DORIS_CHECK(field != nullptr); + if (field->children.empty()) { + return field->physical_column_index == physical_column_index ? field : nullptr; + } + for (auto& child : field->children) { + if (auto* result = find_physical_leaf(&child, physical_column_index); result != nullptr) { + return result; + } + } + return nullptr; +} + +} // namespace + +CountColumnReader::CountColumnReader(std::string name, + std::unique_ptr level_reader, + ParquetColumnReaderProfile profile) + : _level_reader(std::move(level_reader)), _profile(profile), _name(std::move(name)) {} + +CountColumnReader::~CountColumnReader() { + sync_profile(); +} + +Status CountColumnReader::create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(reader != nullptr); + const ParquetColumnSchema* leaf_schema = nullptr; + RETURN_IF_ERROR(find_count_leaf(root_schema, projection, &leaf_schema)); + DORIS_CHECK(leaf_schema != nullptr); + DORIS_CHECK(leaf_schema->leaf_column_id >= 0); + + const auto& thrift_metadata = metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::InvalidArgument("Invalid Parquet COUNT row group {}", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + if (leaf_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet COUNT leaf {} for row group {}", + leaf_schema->leaf_column_id, row_group_id); + } + + DORIS_CHECK(root_schema.local_id >= 0); + auto* root_field = + const_cast(metadata->schema().get_column(root_schema.local_id)); + DORIS_CHECK(root_field != nullptr); + auto* leaf_field = find_physical_leaf(root_field, leaf_schema->leaf_column_id); + if (leaf_field == nullptr) { + return Status::Corruption("Cannot resolve Parquet COUNT physical leaf {} for column {}", + leaf_schema->leaf_column_id, root_schema.name); + } + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + std::unique_ptr level_reader; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + RETURN_IF_ERROR(native::LevelReader::create( + std::move(file), row_group.columns[leaf_schema->leaf_column_id], leaf_field, + row_group.num_rows, std::min(max_group_buffer, max_column_buffer), io_ctx, + enable_page_cache, page_cache_file_key, compat, &level_reader)); + reader->reset(new CountColumnReader(leaf_schema->name, std::move(level_reader), profile)); + return Status::OK(); +} + +Status CountColumnReader::skip(int64_t rows) { + DORIS_CHECK(rows >= 0); + if (rows == 0) { + return Status::OK(); + } + { + SCOPED_TIMER(_profile.level_only_skip_time); + RETURN_IF_ERROR(_level_reader->skip_rows(static_cast(rows))); + } + if (_profile.reader_skip_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_skip_rows, rows); + } + sync_profile(); + return Status::OK(); +} + +Status CountColumnReader::read_levels(int64_t rows, int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(rows_read != nullptr); + _definition_levels.clear(); + _repetition_levels.clear(); + _levels_written = 0; + size_t native_rows_read = 0; + { + SCOPED_TIMER(_profile.level_only_read_time); + RETURN_IF_ERROR(_level_reader->read_rows(static_cast(rows), &_repetition_levels, + &_definition_levels, &native_rows_read)); + } + *rows_read = static_cast(native_rows_read); + if (*rows_read != rows || _definition_levels.size() != _repetition_levels.size()) { + return Status::Corruption( + "Parquet COUNT level reader returned {} rows and {}/{} levels for {}", *rows_read, + _definition_levels.size(), _repetition_levels.size(), _name); + } + _levels_written = static_cast(_definition_levels.size()); + if (_levels_written < *rows_read) { + return Status::Corruption("Parquet COUNT returned {} levels for {} rows in column {}", + _levels_written, *rows_read, _name); + } + if (_profile.reader_read_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_read_rows, *rows_read); + } + sync_profile(); + return Status::OK(); +} + +void CountColumnReader::sync_profile() { + if (_level_reader == nullptr) { + return; + } + const auto stats = _level_reader->statistics(); + const auto& reported = _reported_native_stats; +#define UPDATE_NATIVE_PROFILE(counter, field) \ + if (_profile.counter != nullptr) { \ + COUNTER_UPDATE(_profile.counter, stats.field - reported.field); \ + } + UPDATE_NATIVE_PROFILE(decompress_time, decompress_time); + UPDATE_NATIVE_PROFILE(decompress_count, decompress_cnt); + UPDATE_NATIVE_PROFILE(decode_header_time, decode_header_time); + UPDATE_NATIVE_PROFILE(decode_value_time, decode_value_time); + UPDATE_NATIVE_PROFILE(decode_dictionary_time, decode_dict_time); + UPDATE_NATIVE_PROFILE(decode_level_time, decode_level_time); + UPDATE_NATIVE_PROFILE(skip_page_header_count, skip_page_header_num); + UPDATE_NATIVE_PROFILE(parse_page_header_count, parse_page_header_num); + UPDATE_NATIVE_PROFILE(read_page_header_time, read_page_header_time); + UPDATE_NATIVE_PROFILE(page_read_count, page_read_counter); + UPDATE_NATIVE_PROFILE(page_cache_write_count, page_cache_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_write_count, page_cache_compressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_write_count, + page_cache_decompressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_hit_count, page_cache_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_miss_count, page_cache_missing_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_hit_count, page_cache_compressed_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_hit_count, page_cache_decompressed_hit_counter); +#undef UPDATE_NATIVE_PROFILE + _reported_native_stats = stats; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.h b/be/src/format_v2/parquet/reader/count_column_reader.h new file mode 100644 index 00000000000000..789da8534d2ce3 --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris { +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet { +class NativeParquetMetadata; +struct ParquetColumnSchema; +// Shape-only COUNT(nullable_col) reader. It uses v2's native LevelReader, so BYTE_ARRAY payloads +// are skipped in the encoding stream and never copied into Arrow builders or Doris strings. +class CountColumnReader { +public: + ~CountColumnReader(); + + static Status create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + Status skip(int64_t rows); + Status read_levels(int64_t rows, int64_t* rows_read); + + const std::vector& definition_levels() const { return _definition_levels; } + const std::vector& repetition_levels() const { return _repetition_levels; } + int64_t levels_written() const { return _levels_written; } + +private: + CountColumnReader(std::string name, std::unique_ptr level_reader, + ParquetColumnReaderProfile profile); + void sync_profile(); + + std::unique_ptr _level_reader; + ParquetColumnReaderProfile _profile; + std::string _name; + std::vector _definition_levels; + std::vector _repetition_levels; + native::ColumnChunkReaderStatistics _reported_native_stats; + int64_t _levels_written = 0; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.cpp b/be/src/format_v2/parquet/reader/list_column_reader.cpp deleted file mode 100644 index c042fc99b512aa..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.cpp +++ /dev/null @@ -1,230 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/list_column_reader.h" - -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status ListColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status ListColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status ListColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_batch(rows); -} - -Status ListColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_levels_batch(rows); -} - -Status ListColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet list build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status ListColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet list process result pointer for column {}", - _name); - } - DORIS_CHECK(_element_reader != nullptr); - ColumnArray* array_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr nested_column; - if (column != nullptr) { - array_column = array_column_from_output(*column); - DORIS_CHECK(array_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - nested_column = array_column->get_data_ptr()->assert_mutable(); - const auto& element_output_type = - assert_cast(*remove_nullable(_type)).get_nested_type(); - remove_nullable_wrapper_if_not_expected(element_output_type, &nested_column); - } - - const auto& def_levels = _element_reader->nested_definition_levels(); - const auto& rep_levels = _element_reader->nested_repetition_levels(); - const int64_t levels_written = _element_reader->nested_levels_written(); - std::vector entry_counts; - NullMap parent_nulls; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - if (rep_level == _repetition_level) { - if (entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet LIST column {}", - _name); - } - if (def_level >= _definition_level) { - ++entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet LIST column {} contains null for non-nullable LIST", - _name); - } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - int64_t child_value_count = 0; - if (!_element_reader->is_or_has_repeated_child()) { - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - } - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(total_entries), nested_column, &child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(total_entries), &child_value_count)); - } - } else { - uint64_t pending_entries = 0; - auto flush_pending_entries = [&]() -> Status { - if (pending_entries == 0) { - return Status::OK(); - } - int64_t span_child_value_count = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(pending_entries), nested_column, - &span_child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(pending_entries), &span_child_value_count)); - } - if (span_child_value_count != static_cast(pending_entries)) { - return Status::Corruption( - "Parquet LIST column {} built {} child values, expected {}", _name, - span_child_value_count, pending_entries); - } - child_value_count += span_child_value_count; - pending_entries = 0; - return Status::OK(); - }; - - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - if (entry_count > 0) { - pending_entries += entry_count; - continue; - } - RETURN_IF_ERROR(flush_pending_entries()); - _element_reader->advance_nested_build_level_cursor_past_parent(_repetition_level); - } - RETURN_IF_ERROR(flush_pending_entries()); - } - if (child_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet LIST column {} built {} child values, expected {}", - _name, child_value_count, total_entries); - } - if (column != nullptr) { - array_column->get_data_ptr() = std::move(nested_column); - append_offsets(array_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& ListColumnReader::nested_definition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_definition_levels(); -} - -const std::vector& ListColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_repetition_levels(); -} - -int64_t ListColumnReader::nested_levels_written() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_levels_written(); -} - -bool ListColumnReader::is_or_has_repeated_child() const { - return true; -} - -void ListColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_element_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _element_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.h b/be/src/format_v2/parquet/reader/list_column_reader.h deleted file mode 100644 index d64be546e394d7..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class ListColumnReader final : public ParquetColumnReader { -public: - ListColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr element_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _element_reader(std::move(element_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr - _element_reader; // element reader (recursive; may be Scalar/Struct/List/Map) -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.cpp b/be/src/format_v2/parquet/reader/map_column_reader.cpp deleted file mode 100644 index 8217d0c013abc0..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.cpp +++ /dev/null @@ -1,285 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/map_column_reader.h" - -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status MapColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status MapColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status MapColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_batch(rows)); - return _value_reader->load_nested_batch(rows); -} - -Status MapColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_levels_batch(rows)); - return _value_reader->load_nested_levels_batch(rows); -} - -Status MapColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet map build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status MapColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet map process result pointer for column {}", - _name); - } - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ColumnMap* map_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr key_column; - MutableColumnPtr value_column; - if (column != nullptr) { - map_column = map_column_from_output(*column); - DORIS_CHECK(map_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - key_column = map_column->get_keys_ptr()->assert_mutable(); - value_column = map_column->get_values_ptr()->assert_mutable(); - const auto& map_output_type = assert_cast(*remove_nullable(_type)); - remove_nullable_wrapper_if_not_expected(map_output_type.get_key_type(), &key_column); - remove_nullable_wrapper_if_not_expected(map_output_type.get_value_type(), &value_column); - } - - const auto& def_levels = _key_reader->nested_definition_levels(); - const auto& rep_levels = _key_reader->nested_repetition_levels(); - const int64_t levels_written = _key_reader->nested_levels_written(); - - std::vector entry_counts; - std::vector map_level_indices; - NullMap parent_nulls; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - const int64_t current_level_idx = level_idx; - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - map_level_indices.push_back(current_level_idx); - if (rep_level == _repetition_level) { - if (entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet MAP column {}", - _name); - } - if (def_level >= _definition_level) { - ++entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable MAP", - _name); - } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - for (const auto entry_count : entry_counts) { - total_entries += entry_count; - } - int64_t key_value_count = 0; - size_t key_start = 0; - if (column != nullptr) { - key_start = key_column->size(); - RETURN_IF_ERROR(_key_reader->build_nested_column(static_cast(total_entries), - key_column, &key_value_count)); - } else if (auto* scalar_key_reader = dynamic_cast(_key_reader.get())) { - // MAP keys are required even if a projected Doris key type is nullable. Validate each - // actual entry directly from the key level stream while advancing past empty/null maps. - for (const int64_t key_level_idx : map_level_indices) { - if (def_levels[key_level_idx] >= _definition_level) { - RETURN_IF_ERROR(scalar_key_reader->validate_nested_value(key_level_idx, true)); - ++key_value_count; - } - } - scalar_key_reader->set_nested_build_level_cursor(level_idx); - } else { - RETURN_IF_ERROR(_key_reader->consume_nested_column(static_cast(total_entries), - &key_value_count)); - } - if (key_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} keys, expected {}", _name, - key_value_count, total_entries); - } - if (column != nullptr) { - if (const auto* nullable_key_column = check_and_get_column(*key_column); - nullable_key_column != nullptr && - nullable_key_column->has_null(key_start, nullable_key_column->size())) { - return Status::Corruption("Parquet MAP column {} contains null key", _name); - } - } - int64_t value_count = 0; - if (auto* scalar_value_reader = dynamic_cast(_value_reader.get())) { - const auto& value_def_levels = scalar_value_reader->nested_definition_levels(); - const auto& value_rep_levels = scalar_value_reader->nested_repetition_levels(); - const int64_t value_levels_written = scalar_value_reader->nested_levels_written(); - int64_t value_level_idx = scalar_value_reader->nested_build_level_cursor(); - for (const int64_t key_level_idx : map_level_indices) { - while (value_level_idx < value_levels_written && - (value_rep_levels[value_level_idx] > _repetition_level || - value_def_levels[value_level_idx] < min_parent_definition_level || - (value_rep_levels[value_level_idx] >= _repetition_level && - value_def_levels[value_level_idx] < _repeated_ancestor_definition_level))) { - ++value_level_idx; - } - if (value_level_idx >= value_levels_written) { - return Status::Corruption( - "Parquet MAP column {} value stream ended before key stream", _name); - } - // MAP is encoded as a repeated key/value struct. The key stream owns entry existence, - // but the value stream still has one shape slot for every consumed MAP slot. Consume - // value slots in lockstep with key slots so shape-only slots from empty/null maps do - // not become scalar values. - if (value_rep_levels[value_level_idx] != rep_levels[key_level_idx]) { - return Status::Corruption( - "Parquet MAP column {} value repetition level is not aligned with key " - "stream", - _name); - } - if (def_levels[key_level_idx] >= _definition_level) { - if (column != nullptr) { - RETURN_IF_ERROR(scalar_value_reader->append_nested_value(value_level_idx, - value_column)); - } else { - RETURN_IF_ERROR( - scalar_value_reader->validate_nested_value(value_level_idx, false)); - } - ++value_count; - } - ++value_level_idx; - } - scalar_value_reader->set_nested_build_level_cursor(value_level_idx); - } else { - // Complex MAP values own their nested shape below the entry slot, so they recursively - // process exactly one child value for each MAP entry. - if (column != nullptr) { - RETURN_IF_ERROR(_value_reader->build_nested_column(static_cast(total_entries), - value_column, &value_count)); - } else { - RETURN_IF_ERROR(_value_reader->consume_nested_column( - static_cast(total_entries), &value_count)); - } - } - if (value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} values, expected {}", _name, - value_count, total_entries); - } - - if (column != nullptr) { - map_column->get_keys_ptr() = std::move(key_column); - map_column->get_values_ptr() = std::move(value_column); - append_offsets(map_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& MapColumnReader::nested_definition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_definition_levels(); -} - -const std::vector& MapColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_repetition_levels(); -} - -int64_t MapColumnReader::nested_levels_written() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_levels_written(); -} - -bool MapColumnReader::is_or_has_repeated_child() const { - return true; -} - -void MapColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _key_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _value_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.h b/be/src/format_v2/parquet/reader/map_column_reader.h deleted file mode 100644 index 1a8ca9c70d8c5b..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.h +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -// 2. build_nested_column() -> -class MapColumnReader final : public ParquetColumnReader { -public: - MapColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr key_reader, - std::unique_ptr value_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _key_reader(std::move(key_reader)), - _value_reader(std::move(value_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr _key_reader; // key column reader (always read fully) - std::unique_ptr - _value_reader; // value column reader (can be pruned by projection) -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp new file mode 100644 index 00000000000000..3421ecdd296892 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" + +#include + +#include "util/hash_util.hpp" + +namespace doris::format::parquet::native { + +namespace { +Status set_hash_strategy(segment_v2::HashStrategyPB strategy, + std::function* hash_func) { + if (strategy != segment_v2::HashStrategyPB::XX_HASH_64) { + return Status::InvalidArgument("Invalid Parquet Bloom filter hash strategy {}", strategy); + } + *hash_func = [](const void* buf, int64_t len, uint64_t, void* out) { + *reinterpret_cast(out) = + HashUtil::xxhash64_compat_with_seed(reinterpret_cast(buf), len, 0); + }; + return Status::OK(); +} +} // namespace + +Status BlockSplitBloomFilter::init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) { + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _num_bytes = filter_size; + _size = _num_bytes; + _data = new char[_size]; + memset(_data, 0, _size); + _has_null = nullptr; + _is_write = true; + segment_v2::g_write_bloom_filter_num << 1; + segment_v2::g_write_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +Status BlockSplitBloomFilter::init(const char* buf, size_t size, + segment_v2::HashStrategyPB strategy) { + if (buf == nullptr || size <= 1) { + return Status::InvalidArgument("Invalid Parquet Bloom filter buffer of size {}", size); + } + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _data = new char[size]; + memcpy(_data, buf, size); + _size = size; + _num_bytes = size; + _has_null = nullptr; + segment_v2::g_read_bloom_filter_num << 1; + segment_v2::g_read_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +void BlockSplitBloomFilter::add_bytes(const char* buf, size_t size) { + DCHECK(buf != nullptr); + add_hash(hash(buf, size)); +} + +bool BlockSplitBloomFilter::test_bytes(const char* buf, size_t size) const { + return test_hash(hash(buf, size)); +} + +void BlockSplitBloomFilter::set_has_null(bool has_null) { + DCHECK(!has_null) << "Parquet Bloom filters do not track nulls"; +} + +void BlockSplitBloomFilter::set_masks(uint32_t key, BlockMask* block_mask) { + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + block_mask->item[i] = uint32_t {1} << ((key * SALT[i]) >> 27); + } +} + +void BlockSplitBloomFilter::add_hash(uint64_t hash) { + DCHECK_GE(_num_bytes, BYTES_PER_BLOCK); + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + bitset[bucket_index * BITS_SET_PER_BLOCK + i] |= block_mask.item[i]; + } +} + +bool BlockSplitBloomFilter::test_hash(uint64_t hash) const { + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + const auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + if ((bitset[bucket_index * BITS_SET_PER_BLOCK + i] & block_mask.item[i]) == 0) { + return false; + } + } + return true; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h new file mode 100644 index 00000000000000..38dc97712ac46c --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/bloom_filter/bloom_filter.h" + +namespace doris::format::parquet::native { + +// Keep the split-block implementation inside v2; the native scan path must not link through the +// legacy Parquet reader merely to evaluate footer Bloom filters. +class BlockSplitBloomFilter final : public segment_v2::BloomFilter { +public: + Status init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) override; + Status init(const char* buf, size_t size, segment_v2::HashStrategyPB strategy) override; + void add_bytes(const char* buf, size_t size) override; + bool test_bytes(const char* buf, size_t size) const override; + void set_has_null(bool has_null) override; + bool has_null() const override { return false; } + void add_hash(uint64_t hash) override; + bool test_hash(uint64_t hash) const override; + +private: + static constexpr int BYTES_PER_BLOCK = 32; + static constexpr int BITS_SET_PER_BLOCK = 8; + static constexpr uint32_t SALT[BITS_SET_PER_BLOCK] = {0x47b6137bU, 0x44974d91U, 0x8824ad5bU, + 0xa2b7289dU, 0x705495c7U, 0x2df1424bU, + 0x9efc4947U, 0x5c6bfb31U}; + + struct BlockMask { + uint32_t item[BITS_SET_PER_BLOCK]; + }; + + static void set_masks(uint32_t key, BlockMask* block_mask); +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp new file mode 100644 index 00000000000000..06e4b239f3abb4 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" + +#include + +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "util/bit_util.h" + +namespace doris::format::parquet::native { +Status BoolPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + std::array values; + size_t decoded = 0; + while (decoded < num_values) { + const size_t batch_size = std::min(values.size(), num_values - decoded); + for (size_t row = 0; row < batch_size; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain decoder"); + } + values[row] = static_cast(value); + } + RETURN_IF_ERROR(consumer.consume(values.data(), batch_size, sizeof(uint8_t))); + decoded += batch_size; + } + return Status::OK(); +} + +Status BoolPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + selected_values_.resize(selection.selected_values); + size_t range_index = 0; + size_t output = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + selected_values_[output++] = static_cast(value); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(selected_values_.data(), output, sizeof(uint8_t)); +} + +Status BoolPlainDecoder::skip_values(size_t num_values) { + int skip_cached = + std::min(num_unpacked_values_ - unpacked_value_idx_, cast_set(num_values)); + unpacked_value_idx_ += skip_cached; + if (skip_cached == num_values) { + return Status::OK(); + } + int num_remaining = cast_set(num_values - skip_cached); + int num_to_skip = BitUtil::RoundDownToPowerOf2(num_remaining, 32); + if (num_to_skip > 0) { + // A failed bulk skip must not be reported as success for a truncated boolean page. + if (!bool_values_.SkipBatch(1, num_to_skip)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } + } + num_remaining -= num_to_skip; + if (num_remaining > 0) { + DCHECK_LE(num_remaining, UNPACKED_BUFFER_LEN); + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ < num_remaining)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } + unpacked_value_idx_ = num_remaining; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h new file mode 100644 index 00000000000000..ee49a236b847ce --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris { +class ColumnSelectVector; +} // namespace doris + +namespace doris::format::parquet::native { +/// Decoder bit-packed boolean-encoded values. +/// Implementation from https://github.com/apache/impala/blob/master/be/src/exec/parquet/parquet-bool-decoder.h +//bit-packed-run-len and rle-run-len must be in the range [1, 2^31 - 1]. +// This means that a Parquet implementation can always store the run length in a signed 32-bit integer +class BoolPlainDecoder final : public Decoder { +public: + BoolPlainDecoder() = default; + ~BoolPlainDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + bool_values_.Reset((const uint8_t*)data->data, data->size); + num_unpacked_values_ = 0; + unpacked_value_idx_ = 0; + _offset = 0; + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&selected_values_, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return selected_values_.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return selected_values_.size() * sizeof(uint8_t); + } + +protected: + inline bool _decode_value(bool* value) { + if (LIKELY(unpacked_value_idx_ < num_unpacked_values_)) { + *value = unpacked_values_[unpacked_value_idx_++]; + } else { + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ == 0)) { + return false; + } + *value = unpacked_values_[0]; + unpacked_value_idx_ = 1; + } + return true; + } + + /// A buffer to store unpacked values. Must be a multiple of 32 size to use the + /// batch-oriented interface of BatchedBitReader. We use uint8_t instead of bool because + /// bit unpacking is only supported for unsigned integers. The values are converted to + /// bool when returned to the user. + static const int UNPACKED_BUFFER_LEN = 128; + uint8_t unpacked_values_[UNPACKED_BUFFER_LEN]; + + /// The number of valid values in 'unpacked_values_'. + int num_unpacked_values_ = 0; + + /// The next value to return from 'unpacked_values_'. + int unpacked_value_idx_ = 0; + + /// Bit packed decoder, used if 'encoding_' is PLAIN. + BatchedBitReader bool_values_; + + std::vector selected_values_; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp new file mode 100644 index 00000000000000..d541550cef180f --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" + +#include + +#include +#include +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +Status BoolRLEDecoder::set_data(Slice* slice) { + _data = slice; + _num_bytes = slice->size; + _offset = 0; + if (_num_bytes < 4) { + return Status::IOError("Received invalid length : " + std::to_string(_num_bytes) + + " (corrupt data page?)"); + } + // Load the first 4 bytes in little-endian, which indicates the length + const auto* data = reinterpret_cast(_data->data); + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > static_cast(_num_bytes - 4)) { + return Status::IOError("Received invalid number of bytes : " + std::to_string(num_bytes) + + " (corrupt data page?)"); + } + _num_bytes = num_bytes; + auto decoder_data = data + 4; + _decoder = RleBatchDecoder(const_cast(decoder_data), num_bytes, 1); + return Status::OK(); +} + +Status BoolRLEDecoder::skip_values(size_t num_values) { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + // GetBatch reports truncation; RleDecoder::Skip assumes a valid run and can spin forever. + if (_decoder.GetBatch(_values.data(), static_cast(batch_size)) != batch_size) { + return Status::IOError("Can't skip enough booleans in Parquet RLE decoder"); + } + skipped += batch_size; + } + return Status::OK(); +} + +Status BoolRLEDecoder::decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) { + _values.resize(num_values); + if (_decoder.GetBatch(_values.data(), cast_set(num_values)) != num_values) { + return Status::IOError("Can't read enough booleans in Parquet RLE decoder"); + } + return consumer.consume(_values.data(), _values.size(), sizeof(uint8_t)); +} + +Status BoolRLEDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + _values.resize(selection.total_values); + if (_decoder.GetBatch(_values.data(), cast_set(selection.total_values)) != + selection.total_values) { + return Status::IOError("Can't read enough booleans in Parquet RLE selection decoder"); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, + range.count * sizeof(uint8_t)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_values.data(), output, sizeof(uint8_t)); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h new file mode 100644 index 00000000000000..55d7a331bb27ca --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/rle_encoding.h" + +namespace doris { +class ColumnSelectVector; +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +class BoolRLEDecoder final : public Decoder { +public: + BoolRLEDecoder() = default; + ~BoolRLEDecoder() override = default; + + Status set_data(Slice* slice) override; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { return _values.capacity() * sizeof(uint8_t); } + size_t active_scratch_bytes() const override { return _values.size() * sizeof(uint8_t); } + +private: + RleBatchDecoder _decoder; + std::vector _values; + size_t _num_bytes; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp new file mode 100644 index 00000000000000..cc3dc66b3cacdc --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" + +#include "core/custom_allocator.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { +Status ByteArrayDictDecoder::set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + _dict_items.clear(); + _dict = std::move(dict); + if (_dict == nullptr) { + return Status::Corruption("Wrong dictionary data for byte array type, dict is null."); + } + if (UNLIKELY(length < 0)) { + return Status::Corruption("Wrong data length in dictionary"); + } + const size_t dict_length = cast_set(length); + // Every BYTE_ARRAY entry needs a four-byte length prefix; bound metadata-driven reserve first. + if (UNLIKELY(num_values > dict_length / sizeof(uint32_t))) { + return Status::Corruption("Dictionary value count exceeds byte array payload"); + } + _dict_items.reserve(num_values); + size_t offset_cursor = 0; + char* dict_item_address = reinterpret_cast(_dict.get()); + for (size_t i = 0; i < num_values; ++i) { + if (UNLIKELY(offset_cursor > dict_length || + dict_length - offset_cursor < sizeof(uint32_t))) { + return Status::Corruption("Wrong data length in dictionary"); + } + uint32_t l = decode_fixed32_le(_dict.get() + offset_cursor); + offset_cursor += sizeof(uint32_t); + if (UNLIKELY(l > dict_length - offset_cursor)) { + return Status::Corruption("Wrong data length in dictionary"); + } + _dict_items.emplace_back(dict_item_address + offset_cursor, l); + offset_cursor += l; + } + if (offset_cursor != dict_length) { + return Status::Corruption("Wrong dictionary data for byte array type"); + } + ++_dictionary_generation; + return Status::OK(); +} + +Status ByteArrayDictDecoder::decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return binary_consumer.consume(_dict_items.data(), _dict_items.size()); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h new file mode 100644 index 00000000000000..790d9f2e4b65ba --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/string_ref.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteArrayDictDecoder final : public BaseDictDecoder { +public: + ByteArrayDictDecoder() = default; + ~ByteArrayDictDecoder() override = default; + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override; + + size_t dictionary_size() const override { return _dict_items.size(); } + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override; + +protected: + // StringRef entries point into BaseDictDecoder::_dict. The dictionary buffer lives for the + // entire column chunk, so retaining a second copy of every byte is unnecessary. + DorisVector _dict_items; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp new file mode 100644 index 00000000000000..064bcffec8dd1f --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" + +#include +#include +#include + +#include "core/column/column.h" +#include "core/data_type/data_type_nullable.h" +#include "core/string_ref.h" + +namespace doris::format::parquet::native { +namespace { +Status read_length(const Slice* data, size_t* offset, uint32_t* length) { + if (UNLIKELY(*offset > data->size || data->size - *offset < sizeof(uint32_t))) { + return Status::IOError("Can't read byte array length from plain decoder"); + } + *length = decode_fixed32_le(reinterpret_cast(data->data) + *offset); + *offset += sizeof(uint32_t); + return Status::OK(); +} +} // namespace + +Status ByteArrayPlainDecoder::decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) { + _payload_offsets.clear(); + _payload_offsets.reserve(num_values); + _value_offsets.clear(); + _value_offsets.reserve(num_values + 1); + _value_offsets.push_back(0); + for (size_t row = 0; row < num_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain decoder"); + } + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY batch exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); + _offset += length; + } + _value_spans.clear(); + if (num_values != 0) { + _value_spans.push_back({.first = 0, .count = num_values}); + } + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), num_values, _value_spans); +} + +Status ByteArrayPlainDecoder::decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + _payload_offsets.clear(); + _payload_offsets.reserve(selection.selected_values); + _value_offsets.clear(); + _value_offsets.reserve(selection.selected_values + 1); + _value_offsets.push_back(0); + _value_spans.clear(); + _value_spans.reserve(selection.ranges.size()); + size_t range_index = 0; + size_t selected_in_range = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + if (selected_in_range != 0) { + _value_spans.push_back({.first = _payload_offsets.size() - selected_in_range, + .count = selected_in_range}); + selected_in_range = 0; + } + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY selection exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); + ++selected_in_range; + } + _offset += length; + } + if (selected_in_range != 0) { + _value_spans.push_back( + {.first = _payload_offsets.size() - selected_in_range, .count = selected_in_range}); + } + DORIS_CHECK_EQ(_payload_offsets.size(), selection.selected_values); + if (_payload_offsets.empty()) { + return Status::OK(); + } + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), _payload_offsets.size(), + _value_spans); +} + +Status ByteArrayPlainDecoder::skip_values(size_t num_values) { + for (int i = 0; i < num_values; ++i) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't skip enough bytes in plain decoder"); + } + _offset += length; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h new file mode 100644 index 00000000000000..45bc82fe823f1a --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/types.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_util.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris { +template +class ColumnDecimal; +} // namespace doris + +namespace doris::format::parquet::native { +class ByteArrayPlainDecoder final : public Decoder { +public: + ByteArrayPlainDecoder() = default; + ~ByteArrayPlainDecoder() override = default; + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override; + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_payload_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_spans, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return (_payload_offsets.capacity() + _value_offsets.capacity()) * sizeof(uint32_t) + + _value_spans.capacity() * sizeof(ParquetSelectionRange); + } + size_t active_scratch_bytes() const override { + return (_payload_offsets.size() + _value_offsets.size()) * sizeof(uint32_t) + + _value_spans.size() * sizeof(ParquetSelectionRange); + } + +private: + std::vector _payload_offsets; + std::vector _value_offsets; + std::vector _value_spans; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp new file mode 100644 index 00000000000000..caf3b675e2dfd7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" + +#include +#include + +#include "core/column/column_fixed_length_object.h" +#include "util/byte_stream_split.h" + +namespace doris::format::parquet::native { +Status ByteStreamSplitDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + return Status::IOError("Out-of-bounds access in Parquet byte-stream-split decoder"); + } + const int64_t stride = static_cast(_data->size / _type_length); + _decoded_values.resize(byte_size); + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + _offset / _type_length, num_values, stride, _decoded_values.data()); + _offset += byte_size; + return consumer.consume(_decoded_values.data(), num_values, static_cast(_type_length)); +} + +Status ByteStreamSplitDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet byte-stream-split selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError( + "Out-of-bounds access in Parquet byte-stream-split selection decoder"); + } + DORIS_CHECK_EQ(_data->size % value_width, 0); + const int64_t stride = static_cast(_data->size / value_width); + _decoded_values.resize(selection.selected_values * value_width); + size_t output = 0; + const size_t first_row = _offset / value_width; + for (const auto& range : selection.ranges) { + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + first_row + range.first, range.count, stride, + _decoded_values.data() + output * value_width); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + _offset += input_bytes; + return consumer.consume(_decoded_values.data(), output, value_width); +} + +Status ByteStreamSplitDecoder::skip_values(size_t num_values) { + // Check in row units before multiplication so a corrupt skip count cannot wrap back in-bounds. + if (UNLIKELY(_type_length <= 0 || _offset > _data->size || + num_values > (_data->size - _offset) / _type_length)) { + return Status::IOError( + "Out-of-bounds access in parquet data decoder: offset = {}, size = {}", _offset, + _data->size); + } + _offset += _type_length * num_values; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h new file mode 100644 index 00000000000000..dfe1d954b61641 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteStreamSplitDecoder final : public Decoder { +public: + ByteStreamSplitDecoder() = default; + ~ByteStreamSplitDecoder() override = default; + + Status set_data(Slice* data) override { + if (UNLIKELY(_type_length <= 0 || data == nullptr || + data->size % static_cast(_type_length) != 0)) { + // Every byte lane must contain the same number of complete physical values. + return Status::Corruption("Invalid Parquet byte-stream-split page length"); + } + return Decoder::set_data(data); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_decoded_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _decoded_values.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return _decoded_values.size() * sizeof(uint8_t); + } + +private: + std::vector _decoded_values; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp new file mode 100644 index 00000000000000..26afaf15bf4c4c --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -0,0 +1,1951 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_chunk_reader.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "core/column/column.h" +#include "core/column/column_decimal.h" +#include "core/column/column_dictionary.h" +#include "core/column/column_varbinary.h" +#include "core/column/column_vector.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vexpr.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/bit_util.h" +#include "util/block_compression.h" +#include "util/cpu_info.h" +#include "util/unaligned.h" + +namespace cctz { +class time_zone; +} // namespace cctz +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { + +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available) { + return session_cache_enabled && !storage_cache_disabled && cache_available && header_available; +} + +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed) { + const bool is_v2 = header.__isset.data_page_header_v2; + const bool page_is_compressed = + codec != tparquet::CompressionCodec::UNCOMPRESSED && + (!is_v2 || header.data_page_header_v2.is_compressed || data_page_v2_always_compressed); + if (!page_is_compressed && + UNLIKELY(header.compressed_page_size != header.uncompressed_page_size)) { + // An uncompressed payload has one physical representation, so accepting two lengths makes + // cold reads and decompressed cache hits consume different byte boundaries. + return Status::Corruption( + "Uncompressed Parquet page sizes differ: compressed={}, uncompressed={}", + header.compressed_page_size, header.uncompressed_page_size); + } + return Status::OK(); +} + +Status validate_fixed_width_page_size(const tparquet::PageHeader& header, int32_t type_length, + level_t max_rep_level, level_t max_def_level, + bool schema_is_required) { + if (type_length <= 0) { + return Status::OK(); + } + const bool is_v2 = header.__isset.data_page_header_v2; + if (!is_v2 && !header.__isset.data_page_header) { + return Status::OK(); + } + const auto encoding = + is_v2 ? header.data_page_header_v2.encoding : header.data_page_header.encoding; + if (encoding != tparquet::Encoding::PLAIN && + encoding != tparquet::Encoding::BYTE_STREAM_SPLIT) { + return Status::OK(); + } + int32_t num_physical_values = 0; + int64_t level_bytes = 0; + if (is_v2) { + const auto& page = header.data_page_header_v2; + if (UNLIKELY(page.num_values < 0 || page.num_nulls < 0 || + page.num_nulls > page.num_values || page.repetition_levels_byte_length < 0 || + page.definition_levels_byte_length < 0)) { + return Status::Corruption("Parquet data page v2 has invalid value or level counts"); + } + num_physical_values = page.num_values - page.num_nulls; + level_bytes = static_cast(page.repetition_levels_byte_length) + + page.definition_levels_byte_length; + } else { + if (max_rep_level != 0 || max_def_level != 0 || !schema_is_required) { + return Status::OK(); + } + num_physical_values = header.data_page_header.num_values; + } + if (level_bytes > std::numeric_limits::max() || num_physical_values < 0 || + static_cast(num_physical_values) > + (static_cast(std::numeric_limits::max()) - level_bytes) / + static_cast(type_length)) { + return Status::Corruption("Parquet fixed-width PLAIN page byte size overflows"); + } + const int64_t expected = level_bytes + static_cast(num_physical_values) * type_length; + if (UNLIKELY(header.uncompressed_page_size != expected)) { + // V2 exposes null and level extents separately, so fixed-width payload size is known before + // decompression even for optional columns and must gate attacker-controlled allocation. + return Status::Corruption("Parquet fixed-width page has {} uncompressed bytes, expected {}", + header.uncompressed_page_size, expected); + } + return Status::OK(); +} + +Status validate_dictionary_page_size(const tparquet::PageHeader& header, int32_t type_length) { + DORIS_CHECK(header.__isset.dictionary_page_header); + const int32_t num_values = header.dictionary_page_header.num_values; + if (UNLIKELY(num_values < 0 || (num_values == 0 && header.uncompressed_page_size != 0))) { + // An empty dictionary owns no payload; validate before allocating from its untrusted size. + return Status::Corruption("Parquet dictionary has {} values and {} uncompressed bytes", + num_values, header.uncompressed_page_size); + } + if (type_length > 0) { + if (UNLIKELY(static_cast(num_values) > + static_cast(std::numeric_limits::max()) / + static_cast(type_length))) { + return Status::Corruption("Parquet fixed-width dictionary byte size overflows"); + } + const int64_t expected = static_cast(num_values) * type_length; + if (UNLIKELY(header.uncompressed_page_size != expected)) { + // Fixed-width dictionaries have no level section, so reject forged extents before the + // decoder allocates storage based on the untrusted page header. + return Status::Corruption( + "Parquet fixed-width dictionary has {} uncompressed bytes, expected {}", + header.uncompressed_page_size, expected); + } + } + return Status::OK(); +} + +Status validate_compressed_page_size(tparquet::CompressionCodec::type codec, + const Slice& compressed_data, + size_t expected_uncompressed_size) { + if (codec != tparquet::CompressionCodec::SNAPPY) { + return Status::OK(); + } + size_t actual_uncompressed_size = 0; + if (UNLIKELY(!snappy::GetUncompressedLength(compressed_data.data, compressed_data.size, + &actual_uncompressed_size))) { + return Status::Corruption("Invalid Snappy-compressed Parquet page"); + } + if (UNLIKELY(actual_uncompressed_size != expected_uncompressed_size)) { + // Snappy exposes its decoded extent without an output buffer. Check it before trusting the + // page header so malformed variable-width pages cannot force a header-sized allocation. + return Status::Corruption("Snappy Parquet page expands to {} bytes, expected {}", + actual_uncompressed_size, expected_uncompressed_size); + } + return Status::OK(); +} + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by) { + if (created_by.empty()) { + return {}; + } + const ::parquet::ApplicationVersion version(created_by); + return {.parquet_816_padding = + version.VersionLt(::parquet::ApplicationVersion::PARQUET_816_FIXED_VERSION()), + .data_page_v2_always_compressed = version.VersionLt( + ::parquet::ApplicationVersion::PARQUET_CPP_10353_FIXED_VERSION())}; +} + +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range) { + DORIS_CHECK(range != nullptr); + int64_t start = metadata.data_page_offset; + if (metadata.__isset.dictionary_page_offset && metadata.dictionary_page_offset > 0 && + metadata.dictionary_page_offset < start) { + // Some writers use dictionary_page_offset=0 as an absence sentinel. Range validation must + // follow has_dict_page() or the native reader starts at the Parquet magic bytes. + start = metadata.dictionary_page_offset; + } + const int64_t length = metadata.total_compressed_size; + if (UNLIKELY(start < 0 || length < 0)) { + return Status::Corruption("Parquet column chunk has a negative offset or length"); + } + const uint64_t unsigned_start = static_cast(start); + const uint64_t unsigned_length = static_cast(length); + if (UNLIKELY(unsigned_start > file_size || unsigned_length > file_size - unsigned_start)) { + // Thrift range fields are signed and untrusted; validate before converting them to the + // unsigned stream-reader coordinates so overflow cannot wrap back into the file. + return Status::Corruption("Parquet column chunk [{}, {}) exceeds file size {}", start, + unsigned_start + unsigned_length, file_size); + } + size_t bounded_length = static_cast(unsigned_length); + if (parquet_816_padding) { + // parquet-mr before PARQUET-816 under-reported the chunk by up to 100 bytes. Padding stays + // file-bounded and is only enabled for the affected writer versions. + bounded_length += std::min(100, file_size - unsigned_start - unsigned_length); + } + range->offset = static_cast(unsigned_start); + range->length = bounded_length; + return Status::OK(); +} + +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t data_page_offset, int64_t row_count) { + if (index.page_locations.empty() || data_page_offset < 0 || row_count < 0 || + index.page_locations.front().first_row_index != 0 || + index.page_locations.front().offset != data_page_offset || + chunk_range.length > std::numeric_limits::max() - chunk_range.offset) { + return false; + } + // Row indexes alone cannot detect a uniformly shifted OffsetIndex. Anchor its first location + // to the owning metadata so page-to-row mapping cannot silently move by one physical page. + const uint64_t chunk_begin = chunk_range.offset; + const uint64_t chunk_end = chunk_begin + chunk_range.length; + uint64_t previous_end = chunk_begin; + int64_t previous_row = -1; + for (const auto& location : index.page_locations) { + if (location.first_row_index <= previous_row || location.first_row_index >= row_count || + location.offset < 0 || location.compressed_page_size <= 0) { + return false; + } + const uint64_t begin = static_cast(location.offset); + const uint64_t size = static_cast(location.compressed_page_size); + if (begin < chunk_begin || begin < previous_end || begin > chunk_end || + size > chunk_end - begin) { + return false; + } + previous_row = location.first_row_index; + previous_end = begin + size; + } + return true; +} + +namespace { + +class EmptyValueSectionDecoder final : public Decoder { +public: + Status skip_values(size_t num_values) override { + if (UNLIKELY(num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } + return Status::OK(); + } +}; + +Status append_v2_int96_datetime(ColumnDateTimeV2::Container& data, + const ParquetInt96Timestamp& value, + const cctz::time_zone& timezone) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_PER_DAY = 86400000000LL; + static constexpr int64_t MICROS_PER_SECOND = 1000000LL; + + // Arrow normalized out-of-day INT96 nanos before the native V2 path replaced it. Preserve that + // legacy-writer compatibility here; rejecting the carrier loses valid pre-epoch/year-0 values + // already accepted by Doris external-table scans. + const __int128 days = static_cast<__int128>(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + // Truncate the signed nanos field before day normalization. Flooring a negative value first + // changes the historical result by one microsecond. + const __int128 timestamp_micros = days * MICROS_PER_DAY + value.nanos_of_day / 1000; + if (timestamp_micros < std::numeric_limits::min() || + timestamp_micros > std::numeric_limits::max()) { + return Status::DataQualityError("Parquet INT96 timestamp overflows microseconds"); + } + + const int64_t micros = static_cast(timestamp_micros); + int64_t epoch_seconds = micros / MICROS_PER_SECOND; + int64_t micros_of_second = micros % MICROS_PER_SECOND; + if (micros_of_second < 0) { + micros_of_second += MICROS_PER_SECOND; + --epoch_seconds; + } + DateV2Value datetime; + datetime.from_unixtime(epoch_seconds, timezone); + datetime.set_microsecond(static_cast(micros_of_second)); + if (!datetime.is_valid_date()) { + return Status::DataQualityError("Parquet INT96 timestamp is outside the Doris range"); + } + data.push_back(datetime); + return Status::OK(); +} + +class V2Int96DateTimeConsumer final : public ParquetFixedValueConsumer { +public: + V2Int96DateTimeConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state) + : _data(assert_cast(column).get_data()), _state(state) { + static const auto utc = cctz::utc_time_zone(); + _timezone = context.timezone == nullptr ? &utc : context.timezone; + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + const size_t old_size = _data.size(); + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load( + values + row * sizeof(ParquetInt96Timestamp)); + const auto status = append_v2_int96_datetime(_data, value, *_timezone); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + ParquetMaterializationState* _state; + const cctz::time_zone* _timezone = nullptr; +}; + +class RejectV2Int96BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + return Status::NotSupported("INT96 cannot be decoded from binary Parquet values"); + } +}; + +Status read_v2_int96_datetime(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) { + V2Int96DateTimeConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + V2Int96DateTimeConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectV2Int96BinaryConsumer binary_consumer; + const auto dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + return state.materialize_dictionary(column); +} + +Status read_native_or_serde(IColumn& column, const DataTypeSerDe& serde, + ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) { + if (context.dictionary_index_only) { + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return Status::IOError("Dictionary filter requested for a non-dictionary page"); + } + auto* output = check_and_get_column(&column); + if (output == nullptr) { + return Status::InternalError("Dictionary indices require an INT32 output column"); + } + // Dictionary IDs have the same RLE/bit-packed representation for every physical type. + // Decode them before dispatching to a typed SerDe; otherwise a fixed-width SerDe treats + // the IDs as values and the row filter indexes its bitmap with materialized data. + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = output->size(); + auto& indices = output->get_data(); + indices.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(state.dictionary_indices[row] > + static_cast(std::numeric_limits::max()))) { + indices.resize(old_size); + return Status::Corruption("Parquet dictionary index {} exceeds INT32", + state.dictionary_indices[row]); + } + indices[old_size + row] = static_cast(state.dictionary_indices[row]); + } + return Status::OK(); + } + if (context.physical_type == ParquetPhysicalType::INT96 && + check_and_get_column(&column) != nullptr) { + return read_v2_int96_datetime(column, source, context, num_values, state); + } + return serde.read_column_from_parquet(column, source, context, num_values, state); +} + +Status translate_value_encoding(tparquet::Encoding::type encoding, + ParquetValueEncoding* translated) { + DORIS_CHECK(translated != nullptr); + switch (encoding) { + case tparquet::Encoding::PLAIN: + *translated = ParquetValueEncoding::PLAIN; + return Status::OK(); + case tparquet::Encoding::RLE_DICTIONARY: + case tparquet::Encoding::PLAIN_DICTIONARY: + *translated = ParquetValueEncoding::DICTIONARY; + return Status::OK(); + case tparquet::Encoding::RLE: + *translated = ParquetValueEncoding::RLE; + return Status::OK(); + case tparquet::Encoding::BIT_PACKED: + *translated = ParquetValueEncoding::BIT_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_BINARY_PACKED: + *translated = ParquetValueEncoding::DELTA_BINARY_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_LENGTH_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::DELTA_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::BYTE_STREAM_SPLIT: + *translated = ParquetValueEncoding::BYTE_STREAM_SPLIT; + return Status::OK(); + default: + return Status::NotSupported("Unsupported Parquet encoding {}", + tparquet::to_string(encoding)); + } +} + +template +Status decode_selected_values(IColumn& column, const DataTypeSerDe& serde, Decoder& decoder, + const ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector, + int64_t* materialization_time) { + SCOPED_RAW_TIMER(materialization_time); + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + RETURN_IF_ERROR( + read_native_or_serde(column, serde, decoder, context, run_length, state)); + break; + case ColumnSelectVector::NULL_DATA: + column.insert_many_defaults(run_length); + break; + case ColumnSelectVector::FILTERED_CONTENT: + RETURN_IF_ERROR(decoder.skip_values(run_length)); + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + return Status::OK(); +} + +// Presents one sparse page request as an ordinary sequential source to DataTypeSerDe. SerDe is +// entered once per page fragment; the concrete decoder decides whether to gather selected spans, +// batch-decode and compact, or use the cursor-preserving range fallback. +class SelectedDecodeSource final : public ParquetDecodeSource { +public: + SelectedDecodeSource(Decoder& decoder, const ParquetSelection& selection) + : _decoder(decoder), _selection(selection) {} + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_fixed_values(_selection, consumer); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_binary_values(_selection, consumer); + } + + Status skip_values(size_t num_values) override { + return Status::NotSupported("Selected Parquet source cannot be skipped, values={}", + num_values); + } + + bool has_dictionary() const override { return _decoder.has_dictionary(); } + uint64_t dictionary_generation() const override { return _decoder.dictionary_generation(); } + size_t dictionary_size() const override { return _decoder.dictionary_size(); } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return _decoder.decode_dictionary(fixed_consumer, binary_consumer); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_indices(_selection, indices); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_values(_selection, consumer); + } + + bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const override { + // Avoid touching a dictionary larger than L2 for rows already removed by sparse selection. + // Cache-resident or dense batches instead fuse RLE decode and gather, eliminating the + // page-sized intermediate ID vector. + return _selection.selected_values < _selection.total_values && + dictionary_bytes > static_cast(CpuInfo::get_l2_cache_size()); + } + +private: + Decoder& _decoder; + const ParquetSelection& _selection; +}; + +Status decode_selected_non_null_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = select_vector.num_values(); + selection.selected_values = 0; + + size_t cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + DORIS_CHECK(read_type == ColumnSelectVector::CONTENT || + read_type == ColumnSelectVector::FILTERED_CONTENT); + if (read_type == ColumnSelectVector::CONTENT) { + selection.ranges.push_back({.first = cursor, .count = run_length}); + selection.selected_values += run_length; + } + cursor += run_length; + } + DORIS_CHECK_EQ(cursor, selection.total_values); + if (selection.selected_values == 0) { + return decoder.skip_values(selection.total_values); + } + + SCOPED_RAW_TIMER(materialization_time); + SelectedDecodeSource selected_source(decoder, selection); + return read_native_or_serde(column, serde, selected_source, context, selection.selected_values, + state); +} + +template +bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) { +#define VISIT_COLUMN(TYPE) \ + if (auto* typed = check_and_get_column(&column)) { \ + visitor(*typed); \ + return true; \ + } + VISIT_COLUMN(ColumnUInt8) + VISIT_COLUMN(ColumnInt8) + VISIT_COLUMN(ColumnInt16) + VISIT_COLUMN(ColumnInt32) + VISIT_COLUMN(ColumnInt64) + VISIT_COLUMN(ColumnInt128) + VISIT_COLUMN(ColumnDate) + VISIT_COLUMN(ColumnDateTime) + VISIT_COLUMN(ColumnDateV2) + VISIT_COLUMN(ColumnDateTimeV2) + VISIT_COLUMN(ColumnFloat32) + VISIT_COLUMN(ColumnFloat64) + VISIT_COLUMN(ColumnIPv4) + VISIT_COLUMN(ColumnIPv6) + VISIT_COLUMN(ColumnTimeV2) + VISIT_COLUMN(ColumnTimeStampTz) + VISIT_COLUMN(ColumnOffset32) + VISIT_COLUMN(ColumnOffset64) + VISIT_COLUMN(ColumnDecimal32) + VISIT_COLUMN(ColumnDecimal64) + VISIT_COLUMN(ColumnDecimal128V2) + VISIT_COLUMN(ColumnDecimal128V3) + VISIT_COLUMN(ColumnDecimal256) + VISIT_COLUMN(ColumnString) + VISIT_COLUMN(ColumnString64) + VISIT_COLUMN(ColumnVarbinary) + VISIT_COLUMN(ColumnDictI32) +#undef VISIT_COLUMN + return false; +} + +template +void expand_nullable_pod_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + auto& data = column.get_data(); + DORIS_CHECK_EQ(data.size(), old_size + compact_values); + data.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + data[old_size + output] = typename ColumnType::value_type {}; + } else { + DORIS_CHECK(source > 0); + --source; + data[old_size + output] = std::move(data[old_size + source]); + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_string_values(ColumnStr& column, size_t old_size, + size_t compact_values, const NullMap& selected_nulls) { + auto& offsets = column.get_offsets(); + DORIS_CHECK_EQ(offsets.size(), old_size + compact_values); + const Offset prefix_end = old_size == 0 ? 0 : offsets[old_size - 1]; + offsets.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] == 0) { + DORIS_CHECK(source > 0); + --source; + offsets[old_size + output] = offsets[old_size + source]; + } else { + offsets[old_size + output] = source == 0 ? prefix_end : offsets[old_size + source - 1]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_pod_values(column, old_size, compact_values, selected_nulls); +} + +template +void expand_nullable_values(ColumnStr& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_string_values(column, old_size, compact_values, selected_nulls); +} + +void remap_nullable_conversion_failures(IColumn::Filter* conversion_failure_null_map, + size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + if (conversion_failure_null_map == nullptr) { + return; + } + DORIS_CHECK(conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + size_t source = compact_values; + // Walk backwards so writing an expanded row cannot overwrite an unread compact failure bit. + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + (*conversion_failure_null_map)[old_size + output] = 1; + } else { + DORIS_CHECK(source > 0); + --source; + (*conversion_failure_null_map)[old_size + output] = + (*conversion_failure_null_map)[old_size + source]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +Status decode_selected_nullable_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, NullMap& selected_nulls, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = 0; + selection.selected_values = 0; + selected_nulls.clear(); + selected_nulls.reserve(select_vector.num_values() - select_vector.num_filtered()); + + size_t physical_cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls.resize_fill(selected_nulls.size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls.resize_fill(selected_nulls.size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls.size(), + select_vector.num_values() - select_vector.num_filtered()); + + const size_t old_size = column.size(); + SCOPED_RAW_TIMER(materialization_time); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(decoder.skip_values(selection.total_values)); + column.insert_many_defaults(selected_nulls.size()); + return Status::OK(); + } + + if (state.conversion_failure_null_map != nullptr) { + DORIS_CHECK(state.conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + memset(state.conversion_failure_null_map->data() + old_size, 0, selection.selected_values); + } + SelectedDecodeSource selected_source(decoder, selection); + const auto status = read_native_or_serde(column, serde, selected_source, context, + selection.selected_values, state); + if (!status.ok()) { + if (state.conversion_failure_null_map != nullptr) { + memcpy(state.conversion_failure_null_map->data() + old_size, selected_nulls.data(), + selected_nulls.size()); + } + return status; + } + DORIS_CHECK_EQ(column.size(), old_size + selection.selected_values); + + remap_nullable_conversion_failures(state.conversion_failure_null_map, old_size, + selection.selected_values, selected_nulls); + const bool expanded = visit_nullable_expandable_column(column, [&](auto& typed_column) { + // Decode into the final nested column compactly, then expand in place. This preserves the + // V2 no-intermediate-column invariant while restoring the nullable sparse row layout. + expand_nullable_values(typed_column, old_size, selection.selected_values, selected_nulls); + }); + DORIS_CHECK(expanded); + return Status::OK(); +} + +class FixedWidthPredicateConsumer final : public ParquetFixedValueConsumer { +public: + FixedWidthPredicateConsumer(const VExprSPtrs& conjuncts, DataTypePtr data_type, int column_id, + IColumn::Filter* matches, IColumn* projected_column) + : _conjuncts(conjuncts), + _data_type(std::move(data_type)), + _column_id(column_id), + _matches(matches), + _projected_column(projected_column) { + DORIS_CHECK(_matches != nullptr); + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _matches->size(); + _matches->resize_fill(old_size + num_values, 1); + for (const auto& conjunct : _conjuncts) { + RETURN_IF_ERROR(conjunct->execute_on_raw_fixed_values(values, num_values, value_width, + _data_type, _column_id, + _matches->data() + old_size)); + } + if (_projected_column != nullptr) { + size_t row = 0; + while (row < num_values) { + while (row < num_values && (*_matches)[old_size + row] == 0) { + ++row; + } + const size_t run_begin = row; + while (row < num_values && (*_matches)[old_size + row] != 0) { + ++row; + } + if (row != run_begin) { + _projected_column->insert_many_raw_data( + reinterpret_cast(values + run_begin * value_width), + row - run_begin); + } + } + } + return Status::OK(); + } + +private: + const VExprSPtrs& _conjuncts; + DataTypePtr _data_type; + int _column_id; + IColumn::Filter* _matches; + IColumn* _projected_column; +}; + +} // namespace + +template +ColumnChunkReader::ColumnChunkReader( + io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_rows, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range) + : _field_schema(field_schema), + _max_rep_level(field_schema->repetition_level), + _max_def_level(field_schema->definition_level), + _stream_reader(reader), + _metadata(column_chunk->meta_data), + _offset_index(offset_index), + _total_rows(total_rows), + _io_ctx(io_ctx), + _page_read_ctx(page_read_ctx) { + if (chunk_range != nullptr) { + _chunk_range = *chunk_range; + _has_validated_chunk_range = true; + } +} + +template +Status ColumnChunkReader::init() { + size_t start_offset = _has_validated_chunk_range + ? _chunk_range.offset + : (has_dict_page(_metadata) ? _metadata.dictionary_page_offset + : _metadata.data_page_offset); + size_t chunk_size = + _has_validated_chunk_range ? _chunk_range.length : _metadata.total_compressed_size; + // create page reader + _page_reader = create_page_reader( + _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _metadata, + _page_read_ctx, _offset_index); + // get the block compression codec + RETURN_IF_ERROR(get_block_compression_codec(_metadata.codec, &_block_compress_codec)); + _state = INITIALIZED; + RETURN_IF_ERROR(_parse_first_page_header()); + return Status::OK(); +} + +template +Status ColumnChunkReader::skip_nested_values( + const std::vector& def_levels, size_t start_index) { + size_t no_value_cnt = 0; + size_t value_cnt = 0; + + DORIS_CHECK(start_index <= def_levels.size()); + for (size_t idx = start_index; idx < def_levels.size(); idx++) { + level_t def_level = def_levels[idx]; + if (IN_COLLECTION && def_level < _field_schema->repeated_parent_def_level) { + no_value_cnt++; + } else if (def_level < _field_schema->definition_level) { + no_value_cnt++; + } else { + value_cnt++; + } + } + + RETURN_IF_ERROR(skip_values(value_cnt, true)); + RETURN_IF_ERROR(skip_values(no_value_cnt, false)); + return Status::OK(); +} + +template +Status ColumnChunkReader::read_levels( + size_t num_values, std::vector* rep_levels, std::vector* def_levels) { + DORIS_CHECK(rep_levels != nullptr); + DORIS_CHECK(def_levels != nullptr); + if (_remaining_num_values < num_values || _remaining_rep_nums < num_values || + _remaining_def_nums < num_values) { + return Status::Corruption( + "Parquet level reader requested {} slots with only {}/{}/{} remaining", num_values, + _remaining_num_values, _remaining_rep_nums, _remaining_def_nums); + } + + const size_t start_index = def_levels->size(); + rep_levels->resize(rep_levels->size() + num_values, 0); + def_levels->resize(def_levels->size() + num_values, 0); + if (_max_rep_level > 0) { + const size_t decoded = _rep_level_decoder.get_levels( + rep_levels->data() + rep_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet repetition level stream ended after {} of {} slots", + decoded, num_values); + } + } + if (_max_def_level > 0) { + const size_t decoded = _def_level_decoder.get_levels( + def_levels->data() + def_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet definition level stream ended after {} of {} slots", + decoded, num_values); + } + } + _remaining_rep_nums -= num_values; + _remaining_def_nums -= num_values; + return skip_nested_values(*def_levels, start_index); +} + +template +Status ColumnChunkReader::_parse_first_page_header() { + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + _state = INITIALIZED; + return parse_page_header(); + } + if (header->type != tparquet::PageType::DICTIONARY_PAGE) { + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + _state = INITIALIZED; + continue; + } + // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, + // so we should parse the directory page in next_page() + RETURN_IF_ERROR(_decode_dict_page()); + // parse the real first data page + RETURN_IF_ERROR(_page_reader->dict_next_page()); + _state = INITIALIZED; + // A dictionary is the only non-data page with decoder state. Any following index or + // extension pages are skipped by the same pre-data loop. + } +} + +template +Status ColumnChunkReader::parse_page_header() { + if (_state == HEADER_PARSED || _state == DATA_LOADED) { + return Status::OK(); + } + const tparquet::PageHeader* header = nullptr; + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + break; + } + if (header->type == tparquet::PageType::DICTIONARY_PAGE) { + return Status::Corruption("Parquet dictionary page appears after data pages"); + } + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + } + int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values + : header->data_page_header.num_values; + if constexpr (IN_COLLECTION && OFFSET_INDEX) { + if (!_page_reader->is_header_v2() && _page_reader->has_active_offset_index()) { + // V1 nested pages do not declare their logical row count. An OffsetIndex span cannot + // be trusted until repetition levels are decoded, so keep the sequential cursor path. + _page_reader->discard_offset_index(); + _offset_index = nullptr; + } + } + const bool active_offset_index = _page_reader->has_active_offset_index(); + if (page_num_values < 0 || page_num_values > _metadata.num_values || + (!active_offset_index && + static_cast(page_num_values) > + static_cast(_metadata.num_values) - _chunk_parsed_values)) { + // Page counts are untrusted and feed both level decoders and scratch sizing. Bound each + // page by the column metadata before converting to unsigned counters. + return Status::Corruption("Parquet data page value count {} exceeds column total {}", + page_num_values, _metadata.num_values); + } + if constexpr (!IN_COLLECTION) { + const size_t page_start_row = _page_reader->start_row(); + const size_t page_end_row = _page_reader->end_row(); + if (UNLIKELY(page_end_row < page_start_row || + static_cast(page_num_values) != page_end_row - page_start_row)) { + // Flat columns have exactly one physical value slot per logical row. Rejecting a + // divergent header/OffsetIndex span prevents every later page from shifting rows. + return Status::Corruption( + "Parquet flat data page has {} values for logical row range [{}, {})", + page_num_values, page_start_row, page_end_row); + } + } else if (_page_reader->is_header_v2() && active_offset_index) { + const size_t page_start_row = _page_reader->start_row(); + const size_t page_end_row = _page_reader->end_row(); + if (UNLIKELY(page_end_row < page_start_row || + static_cast(header->data_page_header_v2.num_rows) != + page_end_row - page_start_row)) { + // V2 is the only repeated-page format that states its logical row count in the page + // header, so it must agree with the optional index before indexed seeking is allowed. + return Status::Corruption( + "Parquet nested data page has {} rows for indexed row range [{}, {})", + header->data_page_header_v2.num_rows, page_start_row, page_end_row); + } + } + _remaining_rep_nums = page_num_values; + _remaining_def_nums = page_num_values; + _remaining_num_values = page_num_values; + + // no offset will parse all header. + if (!active_offset_index) { + _chunk_parsed_values += _remaining_num_values; + } + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status ColumnChunkReader::next_page() { + // Level parsing advances _page_data past the allocation base, so retain explicit ownership + // state instead of inferring whether current decoders still reference decompressed storage. + _page_uses_decompress_buf = false; + _active_decompress_bytes = 0; + if (_decompress_release_pending) { + if (_decompress_buf_size > _decompress_release_threshold) { + _decompress_buf.reset(); + _decompress_buf_size = 0; + } + _decompress_release_pending = false; + _decompress_release_threshold = std::numeric_limits::max(); + } + _state = INITIALIZED; + RETURN_IF_ERROR(_page_reader->next_page()); + return Status::OK(); +} + +template +Status ColumnChunkReader::_get_uncompressed_levels( + const tparquet::DataPageHeaderV2& page_v2, Slice& page_data) { + const size_t rl = page_v2.repetition_levels_byte_length; + const size_t dl = page_v2.definition_levels_byte_length; + if (UNLIKELY(rl > page_data.size || dl > page_data.size - rl)) { + // Validate the physical slice again because a cached entry may itself be truncated. + return Status::Corruption("Parquet data page v2 level bytes exceed available payload"); + } + _v2_rep_levels = Slice(page_data.data, rl); + _v2_def_levels = Slice(page_data.data + rl, dl); + page_data.data += dl + rl; + page_data.size -= dl + rl; + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_data() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::Corruption("Should parse page header"); + } + + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); + // Zero levels alone are insufficient: test/protocol adapters can leave repetition unset, so + // only an explicitly REQUIRED schema proves that every logical value has fixed-width bytes. + const bool schema_is_required = _field_schema->parquet_schema.__isset.repetition_type && + _field_schema->parquet_schema.repetition_type == + tparquet::FieldRepetitionType::REQUIRED; + RETURN_IF_ERROR(validate_fixed_width_page_size(*header, _get_type_length(), _max_rep_level, + _max_def_level, schema_is_required)); + int32_t uncompressed_size = header->uncompressed_page_size; + bool page_loaded = false; + _page_uses_decompress_buf = false; + _active_decompress_bytes = 0; + + // First, try to reuse a cache handle previously discovered by PageReader + // (header-only lookup) to avoid a second lookup here. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + size_t levels_size = 0; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + levels_size = rl + dl; + if (UNLIKELY(header_size > cached.size || + levels_size > cached.size - header_size)) { + return Status::Corruption("Cached Parquet page is shorter than its v2 levels"); + } + _v2_rep_levels = + Slice(reinterpret_cast(cached.data) + header_size, rl); + _v2_def_levels = + Slice(reinterpret_cast(cached.data) + header_size + rl, dl); + } + // payload_slice points to the bytes after header and levels + if (UNLIKELY(header_size + levels_size > cached.size)) { + return Status::Corruption("Cached Parquet page is shorter than its header"); + } + Slice payload_slice(cached.data + header_size + levels_size, + cached.size - header_size - levels_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + const size_t expected_payload_size = + cache_payload_is_decompressed + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->compressed_page_size) - levels_size; + if (UNLIKELY(payload_slice.size != expected_payload_size)) { + return Status::Corruption("Cached Parquet page payload has size {}, expected {}", + payload_slice.size, expected_payload_size); + } + + if (cache_payload_is_decompressed) { + // Cached payload is already uncompressed + _page_data = payload_slice; + } else { + CHECK(_block_compress_codec); + // Decompress cached payload into _decompress_buf for decoding + size_t uncompressed_payload_size = + header->__isset.data_page_header_v2 + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->uncompressed_page_size); + RETURN_IF_ERROR(validate_compressed_page_size(_metadata.codec, payload_slice, + uncompressed_payload_size)); + _reserve_decompress_buf(uncompressed_payload_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_payload_size); + _page_uses_decompress_buf = true; + _active_decompress_bytes = uncompressed_payload_size; + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &_page_data)); + if (UNLIKELY(_page_data.size != uncompressed_payload_size)) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_payload_size); + } + } + // page cache counters were incremented when PageReader did the header-only + // cache lookup. Do not increment again to avoid double-counting. + page_loaded = true; + } + } + + if (!page_loaded) { + const bool prepare_cache_payload = can_prepare_page_cache_payload( + _page_read_ctx.enable_parquet_file_page_cache, config::disable_storage_page_cache, + StoragePageCache::instance() != nullptr, !_page_reader->header_bytes().empty()); + if (_block_compress_codec != nullptr) { + Slice compressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + std::vector level_bytes; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + // uncompressed_size = rl + dl + uncompressed_data_size + // compressed_size = rl + dl + compressed_data_size + uncompressed_size -= header_v2.repetition_levels_byte_length + + header_v2.definition_levels_byte_length; + // copy level bytes (rl + dl) so that we can cache header + levels + uncompressed payload + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (prepare_cache_payload && level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), compressed_data.data, level_sz); + } + // now remove levels from compressed_data for decompression + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, compressed_data)); + } + bool is_v2_compressed = header->__isset.data_page_header_v2 && + (header->data_page_header_v2.is_compressed || + _page_read_ctx.data_page_v2_always_compressed); + bool page_has_compression = header->__isset.data_page_header || is_v2_compressed; + + if (page_has_compression) { + // Decompress payload for immediate decoding + RETURN_IF_ERROR(validate_compressed_page_size( + _metadata.codec, compressed_data, static_cast(uncompressed_size))); + _reserve_decompress_buf(uncompressed_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_size); + _page_uses_decompress_buf = true; + _active_decompress_bytes = static_cast(uncompressed_size); + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data)); + if (UNLIKELY(_page_data.size != static_cast(uncompressed_size))) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_size); + } + + // Decide whether to cache decompressed payload or compressed payload based on threshold + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (prepare_cache_payload) { + if (cache_payload_decompressed) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + // cache the compressed payload as-is (header | levels | compressed_payload) + _insert_page_into_cache( + level_bytes, Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + } else { + // no compression on this page, use the data directly + _page_data = Slice(compressed_data.data, compressed_data.size); + if (prepare_cache_payload) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } else { + // For uncompressed page, we may still need to extract v2 levels + std::vector level_bytes; + Slice uncompressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(uncompressed_data)); + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (prepare_cache_payload && level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), uncompressed_data.data, level_sz); + } + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, uncompressed_data)); + } + // copy page data out + _page_data = Slice(uncompressed_data.data, uncompressed_data.size); + // Optionally cache uncompressed data for uncompressed pages + if (prepare_cache_payload) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + + // Initialize repetition level and definition level. Skip when level = 0, which means required field. + if (_max_rep_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_rep_level_decoder.init_v2(_v2_rep_levels, _max_rep_level, + _remaining_rep_nums)); + } else { + RETURN_IF_ERROR(_rep_level_decoder.init( + &_page_data, header->data_page_header.repetition_level_encoding, _max_rep_level, + _remaining_rep_nums)); + } + } + if (_max_def_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_def_level_decoder.init_v2(_v2_def_levels, _max_def_level, + _remaining_def_nums)); + } else { + RETURN_IF_ERROR(_def_level_decoder.init( + &_page_data, header->data_page_header.definition_level_encoding, _max_def_level, + _remaining_def_nums)); + } + } + auto encoding = header->__isset.data_page_header_v2 ? header->data_page_header_v2.encoding + : header->data_page_header.encoding; + // change the deprecated encoding to RLE_DICTIONARY + if (encoding == tparquet::Encoding::PLAIN_DICTIONARY) { + encoding = tparquet::Encoding::RLE_DICTIONARY; + } + _current_encoding = encoding; + + // Reuse page decoder + Decoder* encoding_decoder = nullptr; + if (_decoders.find(static_cast(encoding)) != _decoders.end()) { + encoding_decoder = _decoders[static_cast(encoding)].get(); + } else { + std::unique_ptr page_decoder; + RETURN_IF_ERROR(Decoder::get_decoder(_metadata.type, encoding, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + _decoders[static_cast(encoding)] = std::move(page_decoder); + encoding_decoder = _decoders[static_cast(encoding)].get(); + } + _empty_value_section = _page_data.empty() && _max_def_level > 0; + if (_empty_value_section) { + // Nullable all-NULL pages legally contain only definition levels. Keep them decodable for + // every advertised encoding, but make a non-NULL definition level fail before stale decoder + // state from the preceding page can be consumed. + if (_empty_value_decoder == nullptr) { + _empty_value_decoder = std::make_unique(); + } + _page_decoder = _empty_value_decoder.get(); + } else { + _page_decoder = encoding_decoder; + // Encoding headers cannot legitimately advertise more physical values than the data page's + // logical value count; establish the bound before decoders inspect external counts. + _page_decoder->set_expected_values(_remaining_num_values); + RETURN_IF_ERROR(_page_decoder->set_data(&_page_data)); + } + + _state = DATA_LOADED; + return Status::OK(); +} + +template +Status ColumnChunkReader::_decode_dict_page() { + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type); + SCOPED_RAW_TIMER(&_chunk_statistics.decode_dict_time); + + // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. + // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files. + // refer: https://github.com/apache/parquet-format/blob/master/Encodings.md + tparquet::Encoding::type dict_encoding = header->dictionary_page_header.encoding; + if (dict_encoding != tparquet::Encoding::PLAIN_DICTIONARY && + dict_encoding != tparquet::Encoding::PLAIN) { + return Status::InternalError("Unsupported dictionary encoding {}", + tparquet::to_string(dict_encoding)); + } + + // Prepare dictionary data + int32_t uncompressed_size = header->uncompressed_page_size; + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); + RETURN_IF_ERROR(validate_dictionary_page_size(*header, _get_type_length())); + DorisUniqueBufferPtr dict_data; + bool dict_buffer_allocated = false; + const auto allocate_dict_buffer = [&]() { + if (!dict_buffer_allocated) { + dict_data = make_unique_buffer(uncompressed_size); + dict_buffer_allocated = true; + } + }; + bool dict_loaded = false; + + // Try to load dictionary page from cache + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + if (UNLIKELY(header_size > cached.size)) { + return Status::Corruption( + "Cached Parquet dictionary is shorter than its page header"); + } + // Dictionary page layout in cache: header | payload (compressed or uncompressed) + Slice payload_slice(cached.data + header_size, cached.size - header_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + + if (cache_payload_is_decompressed) { + // Use cached decompressed dictionary data + if (UNLIKELY(payload_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Cached Parquet dictionary payload has size {}, expected {}", + payload_slice.size, uncompressed_size); + } + allocate_dict_buffer(); + memcpy(dict_data.get(), payload_slice.data, payload_slice.size); + dict_loaded = true; + } else { + CHECK(_block_compress_codec); + // Decompress cached compressed dictionary data + RETURN_IF_ERROR(validate_compressed_page_size( + _metadata.codec, payload_slice, static_cast(uncompressed_size))); + allocate_dict_buffer(); + Slice dict_slice(dict_data.get(), uncompressed_size); + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &dict_slice)); + } + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } + dict_loaded = true; + } + + // When dictionary page is loaded from cache, we need to skip the page data + // to update the offset correctly (similar to calling get_page_data()) + if (dict_loaded) { + _page_reader->skip_page_data(); + } + } + } + + if (!dict_loaded) { + // Load and decompress dictionary page from file + if (_block_compress_codec != nullptr) { + auto dict_num = header->dictionary_page_header.num_values; + Slice compressed_data; + if (dict_num != 0) { + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + RETURN_IF_ERROR(validate_compressed_page_size( + _metadata.codec, compressed_data, static_cast(uncompressed_size))); + allocate_dict_buffer(); + Slice dict_slice(dict_data.get(), uncompressed_size); + // Dictionary probes stop before data pages, so count their decompression here or + // metadata pruning profiles will report no codec work for the scan. + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR( + _block_compress_codec->decompress(compressed_data, &dict_slice)); + } + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } + } + allocate_dict_buffer(); + Slice dict_slice(dict_data.get(), uncompressed_size); + + // Decide whether to cache decompressed or compressed dictionary based on threshold + // If uncompressed_page_size == 0, should_cache_decompressed will return true + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; // Dictionary pages don't have levels + if (cache_payload_decompressed) { + // Cache the decompressed dictionary page + // If dict_num == 0, `dict_slice` will be empty + _insert_page_into_cache(empty_levels, dict_slice); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + DCHECK(!compressed_data.empty()); + // Cache the compressed dictionary page + _insert_page_into_cache(empty_levels, + Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + // `get_page_data` not called, we should skip the page data + // Because `_insert_page_into_cache` will use _page_reader, we should exec `skip_page_data` after `_insert_page_into_cache` + if (dict_num == 0) { + _page_reader->skip_page_data(); + } + } else { + Slice dict_slice; + RETURN_IF_ERROR(_page_reader->get_page_data(dict_slice)); + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption("Parquet dictionary payload has size {}, expected {}", + dict_slice.size, uncompressed_size); + } + allocate_dict_buffer(); + // The data is stored by BufferedStreamReader, we should copy it out + memcpy(dict_data.get(), dict_slice.data, dict_slice.size); + + // Cache the uncompressed dictionary page + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; + Slice payload(dict_data.get(), uncompressed_size); + _insert_page_into_cache(empty_levels, payload); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + allocate_dict_buffer(); + + // Cache page decoder + std::unique_ptr page_decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(_metadata.type, tparquet::Encoding::RLE_DICTIONARY, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + // Set the dictionary data + RETURN_IF_ERROR(page_decoder->set_dict(dict_data, uncompressed_size, + header->dictionary_page_header.num_values)); + _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)] = std::move(page_decoder); + + _has_dict = true; + return Status::OK(); +} + +template +void ColumnChunkReader::_reserve_decompress_buf(size_t size) { + if (size > _decompress_buf_size) { + _decompress_buf_size = BitUtil::next_power_of_two(size); + _decompress_buf = make_unique_buffer(_decompress_buf_size); + } +} + +template +void ColumnChunkReader::_insert_page_into_cache( + const std::vector& level_bytes, const Slice& payload) { + StoragePageCache::CacheKey key = + _page_reader->make_page_cache_key(_page_reader->header_start_offset()); + const std::vector& header_bytes = _page_reader->header_bytes(); + size_t total = header_bytes.size() + level_bytes.size() + payload.size; + auto page = std::make_unique(total, true, segment_v2::DATA_PAGE); + size_t pos = 0; + memcpy(page->data() + pos, header_bytes.data(), header_bytes.size()); + pos += header_bytes.size(); + if (!level_bytes.empty()) { + memcpy(page->data() + pos, level_bytes.data(), level_bytes.size()); + pos += level_bytes.size(); + } + if (payload.size > 0) { + memcpy(page->data() + pos, payload.data, payload.size); + pos += payload.size; + } + page->reset_size(total); + PageCacheHandle handle; + StoragePageCache::instance()->insert(key, page.get(), &handle, segment_v2::DATA_PAGE); + page.release(); + _chunk_statistics.page_cache_write_counter += 1; +} + +template +Status ColumnChunkReader::skip_values(size_t num_values, + bool skip_data) { + if (UNLIKELY(_remaining_num_values < num_values)) { + return Status::IOError("Skip too many values in current page. {} vs. {}", + _remaining_num_values, num_values); + } + if (skip_data) { + if (UNLIKELY(_empty_value_section && num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + RETURN_IF_ERROR(_page_decoder->skip_values(num_values)); + } + // Commit logical page progress only after the physical decoder accepted the whole request. + _remaining_num_values -= num_values; + return Status::OK(); +} + +template +Status ColumnChunkReader::materialize_values( + MutableColumnPtr& doris_column, const DataTypeSerDe& serde, ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector) { + if (select_vector.num_values() == 0) { + return Status::OK(); + } + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + const size_t physical_values = select_vector.num_values() - select_vector.num_nulls(); + if (UNLIKELY(_empty_value_section && physical_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + physical_values); + } + if (UNLIKELY((doris_column->is_column_dictionary() || context.dictionary_index_only) && + !_has_dict && physical_values != 0)) { + return Status::IOError("Not dictionary coded"); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + RETURN_IF_ERROR(translate_value_encoding(_current_encoding, &context.encoding)); + Status status; + if (select_vector.has_filter()) { + if (select_vector.num_nulls() == 0) { + ++_chunk_statistics.hybrid_selection_batches; + status = decode_selected_non_null_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + } else if (visit_nullable_expandable_column(*doris_column, [](auto&) {})) { + // Parquet omits NULL leaf values from the physical stream. For example, the logical + // DATE sequence [d0, NULL, d1, NULL, d2] is physically encoded as [d0, d1, d2]. The + // generic fallback follows the logical selection runs, so those NULLs split one + // contiguous physical span into decode(d0), insert-NULL, decode(d1), insert-NULL, + // decode(d2). On nullable sparse scans this repeatedly enters SerDe and turns decimal + // scaling, date conversion, timestamp/timezone conversion, or dictionary-ID + // materialization into many tiny calls even though the physical values are adjacent. + // + // Build selected non-NULL ranges in physical coordinates instead. In the example this + // decodes [d0, d1, d2] compactly with one SerDe consumer, records [0, 1, 0, 1, 0] as + // the logical NULL layout, and expands the final nested column backwards so unread + // compact values cannot be overwritten. Apply this to every expandable V2 scalar, + // including ordinary PLAIN primitives: otherwise nullable numeric predicates still + // fragment a physical span into millions of SerDe calls and defeat sparse decoding. + ++_chunk_statistics.hybrid_selection_batches; + status = decode_selected_nullable_values( + *doris_column, serde, *_page_decoder, context, state, select_vector, + _nullable_selection_nulls, &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + } else { + ++_chunk_statistics.hybrid_selection_null_fallback_batches; + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); + } + } else { + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, state, + select_vector, + &_chunk_statistics.materialization_time); + } + RETURN_IF_ERROR(status); + _remaining_num_values -= select_vector.num_values(); + return Status::OK(); +} + +template +bool ColumnChunkReader::can_filter_fixed_width_values( + const VExprSPtrs& conjuncts, int column_id) const { + if (conjuncts.empty() || + !supports_raw_fixed_filter_encoding(_current_encoding, _metadata.type)) { + return false; + } + const auto primitive_type = remove_nullable(_field_schema->data_type)->get_primitive_type(); + const bool has_identity_width = + (_metadata.type == tparquet::Type::INT32 && primitive_type == TYPE_INT) || + (_metadata.type == tparquet::Type::INT64 && primitive_type == TYPE_BIGINT) || + (_metadata.type == tparquet::Type::FLOAT && primitive_type == TYPE_FLOAT) || + (_metadata.type == tparquet::Type::DOUBLE && primitive_type == TYPE_DOUBLE); + if (!has_identity_width) { + // Raw predicates consume the physical Parquet width. Logical conversions such as UINT32 + // to BIGINT must stay on the typed path or a four-byte value is interpreted as eight bytes. + return false; + } + return std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); + }); +} + +template +Status ColumnChunkReader::filter_fixed_width_values( + const VExprSPtrs& conjuncts, int column_id, ColumnSelectVector& select_vector, + NullMap* selected_nulls, IColumn::Filter* physical_matches, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter) { + DORIS_CHECK(selected_nulls != nullptr); + DORIS_CHECK(physical_matches != nullptr); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + *used_filter = false; + row_filter->clear(); + if (!can_filter_fixed_width_values(conjuncts, column_id)) { + return Status::OK(); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + + ParquetSelection selection; + selected_nulls->clear(); + selected_nulls->reserve(select_vector.num_values() - select_vector.num_filtered()); + size_t physical_cursor = 0; + auto build_selection = [&]() { + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == + physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls->resize_fill(selected_nulls->size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls->resize_fill(selected_nulls->size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + }; + if (select_vector.has_filter()) { + build_selection.template operator()(); + } else { + build_selection.template operator()(); + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls->size(), + select_vector.num_values() - select_vector.num_filtered()); + if (UNLIKELY(_empty_value_section && selection.total_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + selection.total_values); + } + + physical_matches->clear(); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(_page_decoder->skip_values(selection.total_values)); + } else { + FixedWidthPredicateConsumer consumer(conjuncts, _field_schema->data_type, column_id, + physical_matches, projected_column); + RETURN_IF_ERROR(_page_decoder->decode_selected_fixed_values(selection, consumer)); + DORIS_CHECK_EQ(physical_matches->size(), selection.selected_values); + } + + row_filter->reserve(selected_nulls->size()); + size_t physical_row = 0; + for (const uint8_t is_null : *selected_nulls) { + row_filter->push_back(is_null != 0 ? 0 : (*physical_matches)[physical_row++]); + } + DORIS_CHECK_EQ(physical_row, physical_matches->size()); + // Commit logical progress only after both the raw comparison and NULL remapping succeed. + _remaining_num_values -= select_vector.num_values(); + *used_filter = true; + return Status::OK(); +} + +template +Status ColumnChunkReader::seek_to_nested_row(size_t left_row) { + if constexpr (OFFSET_INDEX) { + if (_page_reader->has_active_offset_index()) { + while (true) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + break; + } else if (has_next_page()) { + RETURN_IF_ERROR(next_page()); + _current_row = _page_reader->start_row(); + } else [[unlikely]] { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } + + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + return Status::OK(); + } + } + + while (true) { + RETURN_IF_ERROR(parse_page_header()); + if (_page_reader->is_header_v2() || !IN_COLLECTION) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + RETURN_IF_ERROR(load_page_data()); + // this page contain this row. + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + break; + } + + _current_row = _page_reader->end_row(); + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } else { + RETURN_IF_ERROR(load_page_data()); + std::vector rep_levels; + std::vector def_levels; + bool cross_page = false; + + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, left_row - _current_row, &result_rows, + &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + bool need_load_next_page = true; + while (cross_page) { + need_load_next_page = false; + rep_levels.clear(); + def_levels.clear(); + RETURN_IF_ERROR(load_cross_page_nested_row(rep_levels, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + } + if (left_row == _current_row) { + break; + } + if (need_load_next_page) { + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } + } + }; + + return Status::OK(); +} + +template +Status ColumnChunkReader::_skip_nested_rows_in_page(size_t num_rows) { + if (num_rows == 0) { + return Status::OK(); + } + + std::vector rep_levels; + std::vector def_levels; + + bool cross_page = false; + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, num_rows, &result_rows, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + DCHECK(cross_page == false); + if (num_rows != result_rows) [[unlikely]] { + return Status::InternalError("no match skip rows, expect {} vs. real {}", num_rows, + result_rows); + } + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_nested_rows( + std::vector& rep_levels, size_t max_rows, size_t* result_rows, bool* cross_page) { + if (_state != DATA_LOADED) [[unlikely]] { + return Status::IOError("Should load page data first to load nested rows"); + } + *cross_page = false; + *result_rows = 0; + // Reserve only the requested row frontier. One nested row may legitimately contain more + // values and grow the vector incrementally, but a forged page count must not allocate gigabytes + // before the level stream proves those values exist. + const size_t requested_frontier = + max_rows == std::numeric_limits::max() ? max_rows : max_rows + 1; + rep_levels.reserve(rep_levels.size() + + std::min(_remaining_rep_nums, requested_frontier)); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } + if constexpr (IN_COLLECTION) { + // A continuation level is valid across later V1 pages only after this chunk has seen + // a row start; accepting it on the first sequential page invents an orphan parent row. + if (!_page_reader->has_active_offset_index() && !_nested_row_started && + rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + if (!_page_reader->has_active_offset_index()) { + _nested_row_started = true; + } + } + if (rep_level == 0) { // rep_level 0 indicates start of new row + if (*result_rows == max_rows) { // this page contain max_rows, page no end. + _current_row += max_rows; + _rep_level_rewind_one(); + return Status::OK(); + } + (*result_rows)++; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + _current_row += *result_rows; + + if ((_page_reader->is_header_v2() || _page_reader->has_active_offset_index()) && + UNLIKELY(_current_row != _page_reader->end_row())) { + // V2 and OffsetIndex advertise an exact logical row span. A page that exhausts its + // repetition levels without that many row starts would otherwise make the caller retry + // the same row forever. + return Status::Corruption( + "Parquet nested data page ended at row {}, expected page end row {}", _current_row, + _page_reader->end_row()); + } + + auto need_check_cross_page = [&]() -> bool { + return IN_COLLECTION && !_page_reader->has_active_offset_index() && + _remaining_rep_nums == 0 && !_page_reader->is_header_v2() && has_next_page(); + }; + *cross_page = need_check_cross_page(); + return Status::OK(); +}; + +template +Status ColumnChunkReader::load_cross_page_nested_row( + std::vector& rep_levels, bool* cross_page) { + RETURN_IF_ERROR(next_page()); + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + + *cross_page = has_next_page(); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } + if constexpr (IN_COLLECTION) { + if (!_page_reader->has_active_offset_index() && !_nested_row_started && + rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + if (!_page_reader->has_active_offset_index()) { + _nested_row_started = true; + } + } + if (rep_level == 0) { // rep_level 0 indicates start of new row + *cross_page = false; + _rep_level_rewind_one(); + break; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + return Status::OK(); +} + +template +int32_t ColumnChunkReader::_get_type_length() { + switch (_field_schema->physical_type) { + case tparquet::Type::INT32: + [[fallthrough]]; + case tparquet::Type::FLOAT: + return 4; + case tparquet::Type::INT64: + [[fallthrough]]; + case tparquet::Type::DOUBLE: + return 8; + case tparquet::Type::INT96: + return 12; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + return _field_schema->parquet_schema.type_length; + default: + return -1; + } +} + +/** + * Checks if the given column has a dictionary page. + * + * This function determines the presence of a dictionary page by checking the + * dictionary_page_offset field in the column metadata. The dictionary_page_offset + * must be set and greater than 0, and it must be less than the data_page_offset. + * + * The reason for these checks is based on the implementation in the Java version + * of ORC, where dictionary_page_offset is used to indicate the absence of a dictionary. + * Additionally, Parquet may write an empty row group, in which case the dictionary page + * content would be empty, and thus the dictionary page should not be read. + * + * See https://github.com/apache/arrow/pull/2667/files + */ +bool has_dict_page(const tparquet::ColumnMetaData& column) { + return column.__isset.dictionary_page_offset && column.dictionary_page_offset > 0 && + column.dictionary_page_offset < column.data_page_offset; +} + +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h new file mode 100644 index 00000000000000..102366f24949ee --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -0,0 +1,416 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/common.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "util/slice.h" + +namespace doris { +class BlockCompressionCodec; +class DataTypeSerDe; +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris::format::parquet::native { +using ::doris::ColumnString; + +struct ColumnChunkRange { + size_t offset = 0; + size_t length = 0; +}; + +struct ParquetReaderCompat { + bool parquet_816_padding = false; + bool data_page_v2_always_compressed = false; +}; + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by); +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range); +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t data_page_offset, int64_t row_count); +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available); +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed); +Status validate_fixed_width_page_size(const tparquet::PageHeader& header, int32_t type_length, + level_t max_rep_level, level_t max_def_level, + bool schema_is_required = true); +Status validate_dictionary_page_size(const tparquet::PageHeader& header, int32_t type_length = -1); +Status validate_compressed_page_size(tparquet::CompressionCodec::type codec, + const Slice& compressed_data, + size_t expected_uncompressed_size); + +struct ColumnChunkReaderStatistics { + int64_t decompress_time = 0; + int64_t decompress_cnt = 0; + int64_t decode_header_time = 0; + int64_t decode_value_time = 0; + int64_t materialization_time = 0; + int64_t hybrid_selection_batches = 0; + int64_t hybrid_selection_ranges = 0; + int64_t hybrid_selection_null_fallback_batches = 0; + int64_t decode_dict_time = 0; + int64_t decode_level_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; +}; + +/** + * Read and decode parquet column data into doris block column. + *

Usage:

+ * // Create chunk reader + * ColumnChunkReader chunk_reader(BufferedStreamReader* reader, + * tparquet::ColumnChunk* column_chunk, + * NativeFieldSchema* fieldSchema); + * // Initialize chunk reader + * chunk_reader.init(); + * while (chunk_reader.has_next_page()) { + * // Seek to next page header. Only read and parse the page header, not page data. + * chunk_reader.next_page(); + * // Load data to decoder. Load the page data into underlying container. + * // Or, we can call the chunk_reader.skip_page() to skip current page. + * chunk_reader.load_page_data(); + * // Decode values into column or slice. + * // Or, we can call chunk_reader.skip_values(num_values) to skip some values. + * chunk_reader.materialize_values(column, serde, context, state, selection); + * } + */ +template +class ColumnChunkReader { +public: + ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_row, io::IOContext* io_ctx, + const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range = nullptr); + ~ColumnChunkReader() = default; + + // Initialize chunk reader, will generate the decoder and codec. + Status init(); + + // Whether the chunk reader has a more page to read. + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + if (_page_reader->has_active_offset_index()) { + return _page_reader->has_next_page(); + } + } + // A nested V1 page can invalidate its offset index after initialization. In that mode the + // validated byte range may contain writer padding, so logical values—not remaining bytes— + // are the authoritative chunk boundary. + return _chunk_parsed_values < _metadata.num_values; + } + + // Skip some values(will not read and parse) in current page if the values are filtered by predicates. + // when skip_data = false, the underlying decoder will not skip data, + // only used when maintaining the consistency of _remaining_num_values. + Status skip_values(size_t num_values, bool skip_data = true); + + // Load page data into the underlying container, + // and initialize the repetition and definition level decoder for current page data. + Status load_page_data(); + Status load_page_data_idempotent() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + return load_page_data(); + } + // The remaining number of values in current page(including null values). Decreased when reading or skipping. + uint32_t remaining_num_values() const { return _remaining_num_values; } + + // Apply one logical selection to the encoded page. Decoder advances physical payload cursors; + // SerDe interprets each selected raw span and materializes the destination Doris column. + Status materialize_values(MutableColumnPtr& doris_column, const DataTypeSerDe& serde, + ParquetDecodeContext& context, ParquetMaterializationState& state, + ColumnSelectVector& select_vector); + + static bool supports_raw_fixed_filter_encoding(tparquet::Encoding::type encoding, + tparquet::Type::type physical_type) { + switch (encoding) { + case tparquet::Encoding::PLAIN: + return physical_type == tparquet::Type::INT32 || + physical_type == tparquet::Type::INT64 || + physical_type == tparquet::Type::FLOAT || + physical_type == tparquet::Type::DOUBLE; + case tparquet::Encoding::BYTE_STREAM_SPLIT: + return physical_type == tparquet::Type::INT32 || + physical_type == tparquet::Type::INT64 || + physical_type == tparquet::Type::FLOAT || + physical_type == tparquet::Type::DOUBLE; + case tparquet::Encoding::DELTA_BINARY_PACKED: + return physical_type == tparquet::Type::INT32 || physical_type == tparquet::Type::INT64; + default: + return false; + } + } + + // Evaluate selected fixed-width values and return one keep byte per selected logical row. + // NULL comparisons are false and therefore never enter the physical consumer; non-null + // matches are appended to projected_column when requested. + Status filter_fixed_width_values(const VExprSPtrs& conjuncts, int column_id, + ColumnSelectVector& select_vector, NullMap* selected_nulls, + IColumn::Filter* physical_matches, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter); + bool can_filter_fixed_width_values(const VExprSPtrs& conjuncts, int column_id) const; + + // Get the repetition level decoder of current page. + LevelDecoder& rep_level_decoder() { return _rep_level_decoder; } + // Get the definition level decoder of current page. + LevelDecoder& def_level_decoder() { return _def_level_decoder; } + + level_t max_rep_level() const { return _max_rep_level; } + level_t max_def_level() const { return _max_def_level; } + + bool has_dict() const { return _has_dict; }; + + // Get page decoder + Decoder* get_page_decoder() { return _page_decoder; } + + void release_decoder_scratch(size_t max_retained_bytes) { + for (auto& [encoding, decoder] : _decoders) { + decoder->release_scratch(max_retained_bytes); + } + // Level decoders may batch-convert unsigned RLE values into Doris' signed level_t. + _rep_level_decoder.release_scratch(max_retained_bytes); + _def_level_decoder.release_scratch(max_retained_bytes); + if (_decompress_buf_size > max_retained_bytes) { + if (_page_uses_decompress_buf) { + // Keep the request until the page boundary because decoders still point into this + // allocation; dropping it now would trade retained memory for a use-after-free. + _decompress_release_pending = true; + _decompress_release_threshold = + std::min(_decompress_release_threshold, max_retained_bytes); + } else { + _decompress_buf.reset(); + _decompress_buf_size = 0; + _decompress_release_pending = false; + _decompress_release_threshold = std::numeric_limits::max(); + } + } + } + + size_t retained_decoder_scratch_bytes() const { + size_t bytes = _decompress_buf_size + _rep_level_decoder.retained_scratch_bytes() + + _def_level_decoder.retained_scratch_bytes(); + for (const auto& [encoding, decoder] : _decoders) { + bytes += decoder->retained_scratch_bytes(); + } + return bytes; + } + + size_t active_decoder_scratch_bytes() const { + // Only the current encoding is active. Old decoder instances retain reusable capacity but + // must not make the high-water policy treat their last batch as current working memory. + return _active_decompress_bytes + + (_page_decoder == nullptr ? 0 : _page_decoder->active_scratch_bytes()) + + _rep_level_decoder.active_scratch_bytes() + + _def_level_decoder.active_scratch_bytes(); + } + + tparquet::Encoding::type current_encoding() const { return _current_encoding; } + + ColumnChunkReaderStatistics& chunk_statistics() { + _chunk_statistics.decode_header_time = _page_reader->page_statistics().decode_header_time; + _chunk_statistics.skip_page_header_num = + _page_reader->page_statistics().skip_page_header_num; + _chunk_statistics.parse_page_header_num = + _page_reader->page_statistics().parse_page_header_num; + _chunk_statistics.read_page_header_time = + _page_reader->page_statistics().read_page_header_time; + // PageReader statistics are already cumulative. Snapshot them instead of folding the same + // totals into the chunk on every FileScannerV2 batch. Cache write counters are owned by + // ColumnChunkReader because insertion happens after decompression and therefore remain in + // the chunk accumulator above. + _chunk_statistics.page_read_counter = _page_reader->page_statistics().page_read_counter; + _chunk_statistics.data_page_read_counter = + _page_reader->page_statistics().data_page_read_counter; + _chunk_statistics.page_cache_hit_counter = + _page_reader->page_statistics().page_cache_hit_counter; + _chunk_statistics.page_cache_missing_counter = + _page_reader->page_statistics().page_cache_missing_counter; + _chunk_statistics.page_cache_compressed_hit_counter = + _page_reader->page_statistics().page_cache_compressed_hit_counter; + _chunk_statistics.page_cache_decompressed_hit_counter = + _page_reader->page_statistics().page_cache_decompressed_hit_counter; + return _chunk_statistics; + } + + Decoder* dictionary_decoder() { + return _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)].get(); + } + + size_t page_start_row() const { return _page_reader->start_row(); } + + size_t page_end_row() const { return _page_reader->end_row(); } + + Status parse_page_header(); + Status next_page(); + + Status seek_to_nested_row(size_t left_row); + // Decode level slots without materializing their values. `read_levels` is the flat-column + // counterpart of load_page_nested_rows()+fill_def(): both advance definition/repetition and + // value streams together so a levels-only consumer cannot desynchronize the next data page. + Status read_levels(size_t num_values, std::vector* rep_levels, + std::vector* def_levels); + Status skip_nested_values(const std::vector& def_levels, size_t start_index = 0); + Status fill_def(std::vector& def_values) { + auto before_sz = def_values.size(); + auto append_sz = _remaining_def_nums - _remaining_rep_nums; + def_values.resize(before_sz + append_sz, 0); + if (max_def_level() != 0) { + auto ptr = def_values.data() + before_sz; + const size_t decoded = _def_level_decoder.get_levels(ptr, append_sz); + if (UNLIKELY(decoded != append_sz)) { + def_values.resize(before_sz); + return Status::Corruption( + "Parquet definition level stream ended after {} of {} slots", decoded, + append_sz); + } + } + _remaining_def_nums -= append_sz; + return Status::OK(); + } + + Status load_page_nested_rows(std::vector& rep_levels, size_t max_rows, + size_t* result_rows, bool* cross_page); + Status load_cross_page_nested_row(std::vector& rep_levels, bool* cross_page); + + Slice get_page_data() const { return _page_data; } + const Slice& v2_rep_levels() const { return _v2_rep_levels; } + const Slice& v2_def_levels() const { return _v2_def_levels; } + ColumnChunkReaderStatistics& statistics() { return chunk_statistics(); } + +private: + enum ColumnChunkReaderState { NOT_INIT, INITIALIZED, HEADER_PARSED, DATA_LOADED, PAGE_SKIPPED }; + + // for check dict page. + Status _parse_first_page_header(); + Status _decode_dict_page(); + + void _reserve_decompress_buf(size_t size); + int32_t _get_type_length(); + void _insert_page_into_cache(const std::vector& level_bytes, const Slice& payload); + + Status _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data); + Status _skip_nested_rows_in_page(size_t num_rows); + + level_t _rep_level_get_next() { + if constexpr (IN_COLLECTION) { + return _rep_level_decoder.get_next(); + } + return 0; + } + + void _rep_level_rewind_one() { + if constexpr (IN_COLLECTION) { + _rep_level_decoder.rewind_one(); + } + } + + ColumnChunkReaderState _state = NOT_INIT; + NativeFieldSchema* _field_schema = nullptr; + const level_t _max_rep_level; + const level_t _max_def_level; + + io::BufferedStreamReader* _stream_reader = nullptr; + tparquet::ColumnMetaData _metadata; + const tparquet::OffsetIndex* _offset_index = nullptr; + size_t _current_row = 0; + size_t _total_rows = 0; + io::IOContext* _io_ctx = nullptr; + + std::unique_ptr> _page_reader; + BlockCompressionCodec* _block_compress_codec = nullptr; + + ParquetPageReadContext _page_read_ctx; + ColumnChunkRange _chunk_range; + bool _has_validated_chunk_range = false; + + LevelDecoder _rep_level_decoder; + LevelDecoder _def_level_decoder; + size_t _chunk_parsed_values = 0; + // this page remaining rep/def nums + // if max_rep_level = 0 / max_def_level = 0, this value retail hava value. + uint32_t _remaining_rep_nums = 0; + uint32_t _remaining_def_nums = 0; + // this page remaining values to be processed (for read/skip). + // need parse this page header. + uint32_t _remaining_num_values = 0; + + Slice _page_data; + DorisUniqueBufferPtr _decompress_buf; + size_t _decompress_buf_size = 0; + bool _page_uses_decompress_buf = false; + size_t _active_decompress_bytes = 0; + bool _decompress_release_pending = false; + size_t _decompress_release_threshold = std::numeric_limits::max(); + Slice _v2_rep_levels; + Slice _v2_def_levels; + bool _dict_checked = false; + bool _has_dict = false; + bool _nested_row_started = false; + Decoder* _page_decoder = nullptr; + std::unique_ptr _empty_value_decoder; + bool _empty_value_section = false; + tparquet::Encoding::type _current_encoding = tparquet::Encoding::PLAIN; + // Map: encoding -> Decoder + // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding + std::unordered_map> _decoders; + NullMap _nullable_selection_nulls; + ColumnChunkReaderStatistics _chunk_statistics; +}; + +bool has_dict_page(const tparquet::ColumnMetaData& column); + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp new file mode 100644 index 00000000000000..592e048846e137 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -0,0 +1,1902 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_reader.h" + +#include +#include +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/column/column.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "exprs/vexpr.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "io/fs/tracing_file_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet::native { +namespace { + +ParquetTimeUnit parquet_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; + } + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; + } + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; + } + return ParquetTimeUnit::UNKNOWN; +} + +template +bool release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + DORIS_CHECK(values != nullptr); + if (values->capacity() * sizeof(T) <= max_retained_bytes) { + return false; + } + std::vector().swap(*values); + return true; +} + +size_t retained_set_bytes(const std::unordered_set& values) { + return values.bucket_count() * sizeof(void*) + values.size() * sizeof(size_t); +} + +Status validate_decimal_physical_type(const NativeFieldSchema& field, int precision, int scale) { + if (precision <= 0 || scale < 0 || scale > precision) { + return Status::Corruption("Parquet decimal field {} has invalid precision {} and scale {}", + field.name, precision, scale); + } + switch (field.physical_type) { + case tparquet::Type::INT32: + if (precision <= 9) { + return Status::OK(); + } + break; + case tparquet::Type::INT64: + if (precision <= 18) { + return Status::OK(); + } + break; + case tparquet::Type::BYTE_ARRAY: + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { + const int length = + field.parquet_schema.__isset.type_length ? field.parquet_schema.type_length : -1; + if (length > 0) { + const int64_t max_precision = (static_cast(length) * 8 - 1) * 30103 / 100000; + if (precision <= max_precision) { + return Status::OK(); + } + } + break; + } + default: + break; + } + return Status::Corruption("Parquet decimal field {} has incompatible physical type {}", + field.name, tparquet::to_string(field.physical_type)); +} + +Status validate_physical_annotation(const NativeFieldSchema& field) { + const auto& schema = field.parquet_schema; + if (field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + (!schema.__isset.type_length || schema.type_length <= 0)) { + return Status::Corruption("Parquet fixed-length field {} has invalid width {}", field.name, + schema.__isset.type_length ? schema.type_length : -1); + } + auto require = [&](bool valid, std::string_view annotation) -> Status { + if (valid) { + return Status::OK(); + } + return Status::Corruption("Parquet {} field {} is incompatible with physical type {}", + annotation, field.name, tparquet::to_string(field.physical_type)); + }; + + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "string"); + } + if (logical.__isset.DECIMAL) { + return validate_decimal_physical_type(field, logical.DECIMAL.precision, + logical.DECIMAL.scale); + } + if (logical.__isset.DATE) { + return require(field.physical_type == tparquet::Type::INT32, "date"); + } + if (logical.__isset.TIME) { + const auto unit = parquet_time_unit(logical.TIME.unit); + return require( + (unit == ParquetTimeUnit::MILLIS && + field.physical_type == tparquet::Type::INT32) || + ((unit == ParquetTimeUnit::MICROS || unit == ParquetTimeUnit::NANOS) && + field.physical_type == tparquet::Type::INT64), + "time"); + } + if (logical.__isset.TIMESTAMP) { + return require( + field.physical_type == tparquet::Type::INT64 && + parquet_time_unit(logical.TIMESTAMP.unit) != ParquetTimeUnit::UNKNOWN, + "timestamp"); + } + if (logical.__isset.INTEGER) { + const int width = logical.INTEGER.bitWidth; + return require(((width == 8 || width == 16 || width == 32) && + field.physical_type == tparquet::Type::INT32) || + (width == 64 && field.physical_type == tparquet::Type::INT64), + "integer"); + } + if (logical.__isset.UUID) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 16, + "UUID"); + } + if (logical.__isset.FLOAT16) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 2, + "FLOAT16"); + } + if (logical.__isset.GEOMETRY || logical.__isset.GEOGRAPHY) { + // Geospatial WKB has no Doris logical mapping here; keep its BYTE_ARRAY payload raw so + // reader validation preserves the physical fallback chosen by native schema inference. + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "geospatial"); + } + if (logical.__isset.UNKNOWN) { + return Status::OK(); + } + return Status::Corruption("Unsupported Parquet logical annotation on field {}", field.name); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "converted string"); + case tparquet::ConvertedType::DECIMAL: + return validate_decimal_physical_type(field, + schema.__isset.precision ? schema.precision : -1, + schema.__isset.scale ? schema.scale : -1); + case tparquet::ConvertedType::DATE: + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + return require(field.physical_type == tparquet::Type::INT32, "converted INT32"); + case tparquet::ConvertedType::TIME_MICROS: + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_64: + return require(field.physical_type == tparquet::Type::INT64, "converted INT64"); + case tparquet::ConvertedType::INTERVAL: + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 12, + "interval"); + default: + return Status::Corruption("Unsupported Parquet converted annotation {} on field {}", + tparquet::to_string(schema.converted_type), field.name); + } +} + +IColumn::Filter* conversion_failure_map(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn::Filter* output_null_map, + IColumn::Filter* compatibility_scratch) { + const auto& schema = field.parquet_schema; + const bool is_utc_timestamp = + field.physical_type == tparquet::Type::INT96 || + (field.physical_type == tparquet::Type::INT64 && + ((schema.__isset.logicalType && schema.logicalType.__isset.TIMESTAMP && + schema.logicalType.TIMESTAMP.isAdjustedToUTC) || + (schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS || + schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MICROS)))); + if (!strict_mode && output_null_map != nullptr && is_utc_timestamp && + remove_nullable(target_type)->get_primitive_type() == TYPE_DATETIMEV2) { + // Legacy UTC timestamp conversion kept out-of-range values as DATETIME defaults. Local + // timestamps intentionally keep non-strict NULL-on-overflow behavior because timezone + // conversion is not part of their representation. + compatibility_scratch->resize_fill(output_null_map->size(), 0); + return compatibility_scratch; + } + return output_null_map; +} + +void mark_local_timestamp_defaults(const NativeFieldSchema& field, const DataTypePtr& target_type, + bool strict_mode, IColumn& data_column, + IColumn::Filter* output_null_map, size_t start_row) { + const auto& schema = field.parquet_schema; + const bool is_local_timestamp = + field.physical_type == tparquet::Type::INT64 && schema.__isset.logicalType && + schema.logicalType.__isset.TIMESTAMP && !schema.logicalType.TIMESTAMP.isAdjustedToUTC; + if (strict_mode || output_null_map == nullptr || !is_local_timestamp || + remove_nullable(target_type)->get_primitive_type() != TYPE_DATETIMEV2) { + return; + } + auto& values = assert_cast(data_column).get_data(); + DORIS_CHECK_EQ(values.size(), output_null_map->size()); + for (size_t row = start_row; row < values.size(); ++row) { + if ((*output_null_map)[row] == 0 && !values[row].is_valid_date()) { + // Local timestamps before Doris' representable calendar can materialize as a zero + // date without a SerDe error. Preserve non-strict scan semantics by nulling only that + // sentinel; physical NULLs and valid local timestamps keep their original map bits. + (*output_null_map)[row] = 1; + } + } +} + +Status init_decode_context(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + DORIS_CHECK(context != nullptr); + // Annotation validation belongs before decoder construction: accepting an impossible pair + // lets the logical SerDe reinterpret a differently sized physical value stream. + RETURN_IF_ERROR(validate_physical_annotation(field)); + switch (field.physical_type) { + case tparquet::Type::BOOLEAN: + context->physical_type = ParquetPhysicalType::BOOLEAN; + break; + case tparquet::Type::INT32: + context->physical_type = ParquetPhysicalType::INT32; + break; + case tparquet::Type::INT64: + context->physical_type = ParquetPhysicalType::INT64; + break; + case tparquet::Type::INT96: + context->physical_type = ParquetPhysicalType::INT96; + break; + case tparquet::Type::FLOAT: + context->physical_type = ParquetPhysicalType::FLOAT; + break; + case tparquet::Type::DOUBLE: + context->physical_type = ParquetPhysicalType::DOUBLE; + break; + case tparquet::Type::BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::BYTE_ARRAY; + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY; + break; + default: + return Status::NotSupported("Unsupported Parquet physical type {}", + tparquet::to_string(field.physical_type)); + } + + const auto& schema = field.parquet_schema; + context->type_length = schema.__isset.type_length ? schema.type_length : -1; + context->decimal_precision = schema.__isset.precision ? schema.precision : -1; + context->decimal_scale = schema.__isset.scale ? schema.scale : -1; + context->timezone = ctz; + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + context->logical_type = ParquetLogicalType::STRING; + } else if (logical.__isset.DECIMAL) { + context->logical_type = ParquetLogicalType::DECIMAL; + context->decimal_precision = logical.DECIMAL.precision; + context->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.DATE) { + context->logical_type = ParquetLogicalType::DATE; + } else if (logical.__isset.TIME) { + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = parquet_time_unit(logical.TIME.unit); + } else if (logical.__isset.TIMESTAMP) { + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = parquet_time_unit(logical.TIMESTAMP.unit); + context->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + } else if (logical.__isset.INTEGER) { + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_bit_width = logical.INTEGER.bitWidth; + context->logical_integer_is_signed = logical.INTEGER.isSigned; + } else if (logical.__isset.UUID) { + context->logical_type = ParquetLogicalType::UUID; + context->logical_uuid = true; + } else if (logical.__isset.FLOAT16) { + context->logical_type = ParquetLogicalType::FLOAT16; + context->logical_float16 = true; + } + if (context->logical_uuid && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 16)) { + return Status::Corruption("Parquet UUID field {} must be FIXED_LEN_BYTE_ARRAY(16)", + field.name); + } + if (context->logical_float16 && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 2)) { + return Status::Corruption("Parquet FLOAT16 field {} must be FIXED_LEN_BYTE_ARRAY(2)", + field.name); + } + return Status::OK(); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + context->logical_type = ParquetLogicalType::STRING; + break; + case tparquet::ConvertedType::DECIMAL: + context->logical_type = ParquetLogicalType::DECIMAL; + break; + case tparquet::ConvertedType::DATE: + context->logical_type = ParquetLogicalType::DATE; + break; + case tparquet::ConvertedType::TIME_MILLIS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MILLIS; + break; + case tparquet::ConvertedType::TIME_MICROS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MICROS; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MILLIS; + // Legacy converted timestamps are defined as UTC-adjusted, unlike an unannotated INT64. + context->timestamp_is_adjusted_to_utc = true; + break; + case tparquet::ConvertedType::TIMESTAMP_MICROS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MICROS; + context->timestamp_is_adjusted_to_utc = true; + break; + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::INT_64: + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_is_signed = + schema.converted_type >= tparquet::ConvertedType::INT_8; + context->logical_integer_bit_width = + schema.converted_type == tparquet::ConvertedType::UINT_8 || + schema.converted_type == tparquet::ConvertedType::INT_8 + ? 8 + : schema.converted_type == tparquet::ConvertedType::UINT_16 || + schema.converted_type == tparquet::ConvertedType::INT_16 + ? 16 + : schema.converted_type == tparquet::ConvertedType::UINT_32 || + schema.converted_type == tparquet::ConvertedType::INT_32 + ? 32 + : 64; + break; + default: + break; + } + return Status::OK(); +} + +} // namespace + +#ifdef BE_TEST +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + return init_decode_context(field, ctz, context); +} + +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode) { + IColumn::Filter output_null_map; + output_null_map.resize_fill(1, 0); + IColumn::Filter compatibility_scratch; + return conversion_failure_map(field, target_type, strict_mode, &output_null_map, + &compatibility_scratch) == &compatibility_scratch; +} + +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row) { + mark_local_timestamp_defaults(field, target_type, strict_mode, data_column, output_null_map, + start_row); +} +#endif + +static void fill_struct_null_map(NativeFieldSchema* field, NullMap& null_map, + const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = def_levels.size(); + DCHECK_EQ(num_levels, rep_levels.size()); + size_t origin_size = null_map.size(); + null_map.resize(origin_size + num_levels); + size_t pos = origin_size; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (def_levels[i] >= field->definition_level) { + null_map[pos++] = 0; + } else { + null_map[pos++] = 1; + } + } + null_map.resize(pos); +} + +static Status fill_array_offset(NativeFieldSchema* field, ColumnArray::Offsets64& offsets_data, + NullMap* null_map_ptr, const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = rep_levels.size(); + if (UNLIKELY(num_levels != def_levels.size())) { + return Status::Corruption("Parquet repetition and definition level counts differ"); + } + size_t origin_size = offsets_data.size(); + offsets_data.resize(origin_size + num_levels); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(origin_size + num_levels); + } + size_t offset_pos = origin_size - 1; + bool parent_opened = false; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (rep_levels[i] == field->repetition_level) { + // A continuation can extend only a parent opened by this aligned logical batch. + if (UNLIKELY(!parent_opened)) { + return Status::Corruption( + "Parquet collection starts with an orphan repetition continuation"); + } + offsets_data[offset_pos]++; + continue; + } + parent_opened = true; + offset_pos++; + offsets_data[offset_pos] = offsets_data[offset_pos - 1]; + if (def_levels[i] >= field->definition_level) { + offsets_data[offset_pos]++; + } + if (null_map_ptr != nullptr) { + if (def_levels[i] >= field->definition_level - 1) { + (*null_map_ptr)[offset_pos] = 0; + } else { + (*null_map_ptr)[offset_pos] = 1; + } + } + } + offsets_data.resize(offset_pos + 1); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(offset_pos + 1); + } + return Status::OK(); +} + +Status ColumnReader::create(io::FileReaderSPtr file, NativeFieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + const std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection, + const std::set& column_ids, + const std::set& filter_column_ids, + const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode) { + size_t total_rows = row_group.num_rows; + if (field->data_type->get_primitive_type() == TYPE_ARRAY) { + std::unique_ptr element_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + element_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, enable_strict_mode)); + auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + element_reader->set_column_in_nested(); + RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field)); + array_reader->_filter_column_ids = filter_column_ids; + reader.reset(array_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_MAP) { + std::unique_ptr key_reader; + std::unique_ptr value_reader; + + if (column_ids.empty() || + column_ids.find(field->children[0].get_column_id()) != column_ids.end()) { + // Create key reader + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + key_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[0]); + key_reader = std::move(skip_reader); + } + + if (column_ids.empty() || + column_ids.find(field->children[1].get_column_id()) != column_ids.end()) { + // Create value reader + RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx, + value_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[1]); + value_reader = std::move(skip_reader); + } + + auto map_reader = MapColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + key_reader->set_column_in_nested(); + value_reader->set_column_in_nested(); + RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field)); + map_reader->_filter_column_ids = filter_column_ids; + reader.reset(map_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) { + std::unordered_map> child_readers; + child_readers.reserve(field->children.size()); + int non_skip_reader_idx = -1; + for (int i = 0; i < field->children.size(); ++i) { + auto& child = field->children[i]; + std::unique_ptr child_reader; + if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) { + RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, + in_collection, column_ids, filter_column_ids, + page_cache_file_key, compat, enable_strict_mode)); + child_readers[child.name] = std::move(child_reader); + // Record the first non-SkippingReader + if (non_skip_reader_idx == -1) { + non_skip_reader_idx = i; + } + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &child); + skip_reader->_filter_column_ids = filter_column_ids; + child_readers[child.name] = std::move(skip_reader); + } + child_readers[child.name]->set_column_in_nested(); + } + // If all children are SkipReadingReader, force the first child to call create + if (non_skip_reader_idx == -1) { + std::unique_ptr child_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, in_collection, + column_ids, filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); + child_reader->set_column_in_nested(); + child_readers[field->children[0].name] = std::move(child_reader); + } + auto struct_reader = StructColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field)); + struct_reader->_filter_column_ids = filter_column_ids; + reader.reset(struct_reader.release()); + } else { + auto physical_index = field->physical_column_index; + if (physical_index < 0 || static_cast(physical_index) >= row_group.columns.size()) { + // Keep the leaf-to-chunk invariant checked at this consumer too because unit callers + // can construct a reader without going through NativeParquetMetadata::init_schema(). + return Status::Corruption("Parquet physical column index {} is out of range {}", + physical_index, row_group.columns.size()); + } + const auto offset_it = col_offsets.find(physical_index); + const tparquet::OffsetIndex* offset_index = + offset_it != col_offsets.end() ? &offset_it->second : nullptr; + + const tparquet::ColumnChunk& chunk = row_group.columns[physical_index]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet physical column {} has no chunk metadata", + physical_index); + } + if (in_collection) { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } else { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } + } + return Status::OK(); +} + +void ColumnReader::_generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const { + result_ranges->add(page_row_range); + RowRanges::ranges_intersection(*result_ranges, _row_ranges, result_ranges); +} + +template +Status ScalarColumnReader::init( + io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, RuntimeState* state, + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, + bool enable_strict_mode) { + _field_schema = field; + auto& chunk_meta = _chunk_meta.meta_data; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(chunk_meta, file->size(), compat.parquet_816_padding, + &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_len = chunk_range.length; + size_t prefetch_buffer_size = std::min(chunk_len, max_buf_size); + if ((typeid_cast(file.get()) && + typeid_cast( + ((doris::io::TracingFileReader*)(file.get()))->inner_reader().get())) || + typeid_cast(file.get())) { + // turn off prefetch data when using MergeRangeFileReader + prefetch_buffer_size = 0; + } + _stream_reader = std::make_unique(file, chunk_start, chunk_len, + prefetch_buffer_size); + ParquetPageReadContext ctx( + (state == nullptr) ? true : state->query_options().enable_parquet_file_page_cache, + page_cache_file_key, compat.data_page_v2_always_compressed); + + _chunk_reader = std::make_unique>( + _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx, ctx, + &chunk_range); + _materialization_state.enable_strict_mode = enable_strict_mode; + RETURN_IF_ERROR(_chunk_reader->init()); + RETURN_IF_ERROR(init_decode_context(*field, _ctz, &_decode_context)); + return Status::OK(); +} + +template +void ScalarColumnReader::release_batch_scratch( + size_t max_retained_bytes) { + const size_t retained_bytes = retained_batch_scratch_bytes(); + const size_t active_bytes = active_batch_scratch_bytes(); + if (retained_bytes <= max_retained_bytes || active_bytes > max_retained_bytes) { + _oversized_scratch_idle_batches = 0; + return; + } + // An adaptive probe or one repeated outlier must not pin memory forever, but immediately + // dropping a large steady-state buffer makes every following batch allocate it again. Require + // three ordinary batches before treating an oversized capacity as idle. + constexpr uint8_t OVERSIZED_SCRATCH_IDLE_BATCHES = 3; + if (++_oversized_scratch_idle_batches < OVERSIZED_SCRATCH_IDLE_BATCHES) { + return; + } + _oversized_scratch_idle_batches = 0; + if (_chunk_reader != nullptr) { + // Persistent decoders also own batch-sized value/slice buffers, not only the reader. + _chunk_reader->release_decoder_scratch(max_retained_bytes); + } + bool release_selection = false; + release_selection |= release_vector_if_oversized(&_rep_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_def_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_null_run_lengths, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_nested_filter_map_data, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.dictionary_indices, + max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.selection.ranges, + max_retained_bytes); + if (retained_set_bytes(_ancestor_null_indices) > max_retained_bytes) { + std::unordered_set().swap(_ancestor_null_indices); + release_selection = true; + } + if (release_selection) { + _select_vector = ColumnSelectVector(); + } +} + +template +size_t ScalarColumnReader::retained_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->retained_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.capacity() * sizeof(level_t) + + _def_levels.capacity() * sizeof(level_t) + + _null_run_lengths.capacity() * sizeof(uint16_t) + + _nested_filter_map_data.capacity() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.capacity() * sizeof(uint32_t) + + _materialization_state.selection.ranges.capacity() * sizeof(ParquetSelectionRange) + + retained_set_bytes(_ancestor_null_indices); +} + +template +size_t ScalarColumnReader::active_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->active_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.size() * sizeof(level_t) + + _def_levels.size() * sizeof(level_t) + _null_run_lengths.size() * sizeof(uint16_t) + + _nested_filter_map_data.size() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.size() * sizeof(uint32_t) + + _materialization_state.selection.ranges.size() * sizeof(ParquetSelectionRange) + + _ancestor_null_indices.size() * sizeof(size_t); +} + +#ifdef BE_TEST +template +void ScalarColumnReader::reserve_batch_scratch_for_test( + size_t elements) { + _rep_levels.reserve(elements); + _def_levels.reserve(elements); + _null_run_lengths.reserve(elements); + _nested_filter_map_data.reserve(elements); + _materialization_state.dictionary_indices.reserve(elements); + _materialization_state.selection.ranges.reserve(elements); + _ancestor_null_indices.reserve(elements); +} + +template +size_t ScalarColumnReader::retained_batch_scratch_bytes_for_test() + const { + return retained_batch_scratch_bytes(); +} +#endif + +template +Status ScalarColumnReader::_skip_values(size_t num_values) { + if (num_values == 0) { + return Status::OK(); + } + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t skipped = 0; + size_t null_size = 0; + size_t nonnull_size = 0; + while (skipped < num_values) { + level_t def_level = -1; + size_t loop_skip = def_decoder.get_next_run(&def_level, num_values - skipped); + if (loop_skip == 0) { + return Status::Corruption("Parquet definition level stream ended while skipping"); + } + if (def_level < _field_schema->definition_level) { + null_size += loop_skip; + } else { + nonnull_size += loop_skip; + } + skipped += loop_skip; + } + if (null_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(null_size, false)); + } + if (nonnull_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(nonnull_size, true)); + } + } else { + RETURN_IF_ERROR(_chunk_reader->skip_values(num_values)); + } + return Status::OK(); +} + +template +Status ScalarColumnReader::_read_values(size_t num_values, + ColumnPtr& doris_column, + const DataTypePtr& type, + FilterMap& filter_map, + bool is_dict_filter) { + if (num_values == 0) { + return Status::OK(); + } + MutableColumnPtr data_column; + _null_run_lengths.clear(); + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level; + size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + return Status::Corruption( + "Parquet definition level stream ended while materializing"); + } + + bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + prev_is_null = is_null; + has_read += loop_read; + } + } + } else { + if (_chunk_reader->max_def_level() > 0) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (_null_run_lengths.empty()) { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + } + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, map_data_column, + &filter_map, _filter_map_index)); + _filter_map_index += num_values; + } + DORIS_CHECK(_serde != nullptr); + // Keep selected-row cardinality stable: non-strict conversion failures append a nested + // default and mark this matching nullable output row instead of shortening the column. + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = + conversion_failure_map(*_field_schema, type, _materialization_state.enable_strict_mode, + map_data_column, &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values(data_column, *_serde, _decode_context, + _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + return status; +} + +/** + * Load the nested column data of complex type. + * A row of complex type may be stored across two(or more) pages, and the parameter `align_rows` indicates that + * whether the reader should read the remaining value of the last row in previous page. + */ +template +Status ScalarColumnReader::_read_nested_column( + ColumnPtr& doris_column, const DataTypePtr& type, FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter) { + _rep_levels.clear(); + _def_levels.clear(); + + // Handle nullable columns + MutableColumnPtr data_column; + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + _nested_filter_map_data.clear(); + + auto read_and_fill_data = [&](size_t before_rep_level_sz, size_t filter_map_index) { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + if (filter_map.has_filter()) { + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, before_rep_level_sz, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + } else { + RETURN_IF_ERROR(_nested_filter_map.init( + nullptr, _rep_levels.size() - before_rep_level_sz, false)); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + RETURN_IF_ERROR(gen_nested_null_map(before_rep_level_sz, _rep_levels.size(), + _null_run_lengths, _ancestor_null_indices)); + + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init( + _null_run_lengths, + _rep_levels.size() - before_rep_level_sz - _ancestor_null_indices.size(), + map_data_column, &_nested_filter_map, 0, &_ancestor_null_indices)); + } + + DORIS_CHECK(_serde != nullptr); + // Nested materialization must preserve the same value/null-map row alignment invariant. + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = conversion_failure_map( + *_field_schema, type, _materialization_state.enable_strict_mode, map_data_column, + &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values( + data_column, *_serde, _decode_context, _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + RETURN_IF_ERROR(status); + if (!_ancestor_null_indices.empty()) { + RETURN_IF_ERROR(_chunk_reader->skip_values(_ancestor_null_indices.size(), false)); + } + if (filter_map.has_filter()) { + auto new_rep_sz = before_rep_level_sz; + for (size_t idx = before_rep_level_sz; idx < _rep_levels.size(); idx++) { + if (_nested_filter_map_data[idx - before_rep_level_sz]) { + _rep_levels[new_rep_sz] = _rep_levels[idx]; + _def_levels[new_rep_sz] = _def_levels[idx]; + new_rep_sz++; + } + } + _rep_levels.resize(new_rep_sz); + _def_levels.resize(new_rep_sz); + } + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + size_t right_row = std::min(left_row + batch_size - *read_rows, + (size_t)_row_ranges.get_range_to(_current_range_idx)); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + size_t load_rows = 0; + bool cross_page = false; + size_t before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &load_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && load_rows == 0 && !cross_page)) { + // A bounded V2/indexed page must advance at least one logical row; zero progress would + // leave both range cursors unchanged and spin forever on corrupt repetition levels. + return Status::Corruption("Parquet nested reader made no row progress"); + } + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index)); + _filter_map_index += load_rows; + while (cross_page) { + before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index - 1)); + } + *read_rows += load_rows; + _current_row_index += load_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Status ScalarColumnReader::_read_fixed_width_filter_values( + size_t num_values, const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, + IColumn* projected_column, IColumn::Filter* row_filter) { + DORIS_CHECK(row_filter != nullptr); + _null_run_lengths.clear(); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level = -1; + const size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + return Status::Corruption( + "Parquet definition level stream ended while filtering fixed-width values"); + } + const bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + prev_is_null = is_null; + has_read += loop_read; + } + } else { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + } + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, nullptr, &filter_map, + _filter_map_index)); + _filter_map_index += num_values; + bool used_filter = false; + RETURN_IF_ERROR(_chunk_reader->filter_fixed_width_values( + conjuncts, column_id, _select_vector, &_fixed_width_predicate_nulls, + &_fixed_width_predicate_matches, projected_column, row_filter, &used_filter)); + // Chunk encodings are prevalidated before any definition level is consumed, so a false result + // here would make a materializing fallback observe an advanced level cursor. + DORIS_CHECK(used_filter); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_fixed_width_filter( + const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, size_t batch_size, + IColumn* projected_column, IColumn::Filter* row_filter, size_t* read_rows, bool* eof, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + if (_in_nested || conjuncts.empty()) { + return Status::OK(); + } + if (!std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); + })) { + return Status::OK(); + } + // Validate every advertised value encoding before touching either cursor. A late fallback + // cannot rewind definition levels or a previously decoded fixed-width page. + const bool supported_encodings = std::ranges::all_of( + _chunk_meta.meta_data.encodings, [&](const tparquet::Encoding::type encoding) { + return ColumnChunkReader:: + supports_raw_fixed_filter_encoding(encoding, + _chunk_meta.meta_data.type) || + encoding == tparquet::Encoding::RLE || + encoding == tparquet::Encoding::BIT_PACKED; + }); + if (!supported_encodings) { + return Status::OK(); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + _current_row_index = right_row; + } else { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + if (!_chunk_reader->can_filter_fixed_width_values(conjuncts, column_id)) { + return Status::OK(); + } + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); ++idx) { + const auto range = read_ranges.get_range(idx); + const size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + const size_t values = + std::min(static_cast(range.to() - range.from()), batch_size - has_read); + IColumn::Filter fragment_filter; + RETURN_IF_ERROR(_read_fixed_width_filter_values( + values, conjuncts, column_id, filter_map, projected_column, &fragment_filter)); + row_filter->insert(row_filter->end(), fragment_filter.begin(), fragment_filter.end()); + has_read += values; + *read_rows += values; + _current_row_index += values; + if (has_read == batch_size) { + break; + } + } + } + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + *used_filter = true; + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_levels(FilterMap& filter_map, + size_t batch_size, + size_t* read_rows, + bool* eof) { + DORIS_CHECK(_in_nested); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + _rep_levels.clear(); + _def_levels.clear(); + *read_rows = 0; + + auto consume_level_segment = [&](size_t level_start, size_t filter_map_index) -> Status { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + // Advance the encoded value stream without constructing a temporary Doris column. The + // definition levels identify the physical values that actually exist; dictionary skips + // still validate every index. + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(_def_levels, level_start)); + if (!filter_map.has_filter()) { + return Status::OK(); + } + + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, level_start, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + size_t write_index = level_start; + for (size_t read_index = level_start; read_index < _rep_levels.size(); ++read_index) { + if (_nested_filter_map_data[read_index - level_start] != 0) { + _rep_levels[write_index] = _rep_levels[read_index]; + _def_levels[write_index] = _def_levels[read_index]; + ++write_index; + } + } + _rep_levels.resize(write_index); + _def_levels.resize(write_index); + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + const size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + const size_t right_row = + std::min(left_row + batch_size - *read_rows, + static_cast(_row_ranges.get_range_to(_current_range_idx))); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + + size_t loaded_rows = 0; + bool cross_page = false; + size_t level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &loaded_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && loaded_rows == 0 && !cross_page)) { + // Keep the levels-only path under the same forward-progress invariant as value reads. + return Status::Corruption("Parquet nested level reader made no row progress"); + } + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index)); + _filter_map_index += loaded_rows; + while (cross_page) { + level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index - 1)); + } + + *read_rows += loaded_rows; + _current_row_index += loaded_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Result +ScalarColumnReader::materialize_dictionary_values( + const ColumnInt32* dict_column, const DataTypePtr& target_type) { + DORIS_CHECK(dict_column != nullptr); + DORIS_CHECK(target_type != nullptr); + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + DORIS_CHECK(dictionary_decoder != nullptr); + // Materialize the dictionary in the file-local logical type once. Both predicate evaluation + // and matched-ID gathering must observe the same decimal scale, timestamp unit, and binary + // interpretation as ordinary data-page decoding. + const DataTypePtr dictionary_type = remove_nullable(target_type); + const DataTypeSerDeSPtr dictionary_serde = dictionary_type->get_serde(); + // The probe is the first typed read for this reader. Publish its SerDe identity now so the + // first dictionary-ID batch does not mistake initialization for a type change and drop cache. + _serde_type = dictionary_type.get(); + _serde = dictionary_serde; + if (_materialization_state.dictionary_generation != + dictionary_decoder->dictionary_generation()) { + _materialization_state.typed_dictionary = dictionary_type->create_column(); + ParquetDecodeContext dictionary_context = _decode_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + dictionary_context.dictionary_index_only = false; + auto status = dictionary_serde->read_parquet_dictionary( + *_materialization_state.typed_dictionary, *dictionary_decoder, dictionary_context); + if (!status.ok()) { + return ResultError(std::move(status)); + } + DORIS_CHECK_EQ(_materialization_state.typed_dictionary->size(), + dictionary_decoder->dictionary_size()); + _materialization_state.dictionary_generation = dictionary_decoder->dictionary_generation(); +#ifdef BE_TEST + ++_dictionary_materialization_count; +#endif + } + + auto result = _materialization_state.typed_dictionary->clone_empty(); + const auto& source_indices = dict_column->get_data(); + auto& indices = _materialization_state.dictionary_indices; + indices.resize(source_indices.size()); + for (size_t row = 0; row < source_indices.size(); ++row) { + if (UNLIKELY(source_indices[row] < 0 || + static_cast(source_indices[row]) >= + _materialization_state.typed_dictionary->size())) { + return ResultError(Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + source_indices[row], row, _materialization_state.typed_dictionary->size())); + } + indices[row] = static_cast(source_indices[row]); + } + result->insert_indices_from(*_materialization_state.typed_dictionary, indices.data(), + indices.data() + indices.size()); + return result; +} + +template +Result ScalarColumnReader::dictionary_values( + const DataTypePtr& target_type) { + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + if (dictionary_decoder == nullptr || dictionary_decoder->dictionary_size() == 0) { + return ResultError(Status::NotSupported("Parquet column has no reusable dictionary")); + } + auto ids = ColumnInt32::create(); + auto& data = ids->get_data(); + data.resize(dictionary_decoder->dictionary_size()); + for (size_t dictionary_id = 0; dictionary_id < data.size(); ++dictionary_id) { + data[dictionary_id] = cast_set(dictionary_id); + } + // Materialize the typed dictionary once and keep it in _materialization_state. Later row-level + // filtering decodes only ids and flattens surviving values from this same dictionary. + return materialize_dictionary_values(ids.get(), target_type); +} + +template +Status ScalarColumnReader::_try_load_dict_page(bool* loaded, + bool* has_dict) { + // _chunk_reader init will load first page header to check whether has dict page + *loaded = true; + *has_dict = _chunk_reader->has_dict(); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_data( + ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, FilterMap& filter_map, + size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + const DataTypePtr materialization_type = remove_nullable(type); + const DataTypePtr file_type = remove_nullable(_field_schema->data_type); + if (!is_dict_filter && !file_type->equals(*materialization_type)) { + // File-to-table casts belong to ColumnMapper. Reaching this reader with a table type would + // also evaluate file-local predicates against values produced under the wrong type. + return Status::InternalError( + "Native Parquet reader type mismatch for file column '{}': file={}, requested={}", + _field_schema->name, file_type->get_name(), materialization_type->get_name()); + } + + const DataTypePtr serde_type = is_dict_filter ? file_type : materialization_type; + const bool serde_type_changed = _serde_type != serde_type.get(); + if (serde_type_changed || _dictionary_index_only != is_dict_filter) { + _serde_type = serde_type.get(); + _serde = serde_type->get_serde(); + _dictionary_index_only = is_dict_filter; + // Switching between typed values and dictionary IDs does not invalidate the dictionary + // materialized by the pruning probe. Preserve it unless the logical SerDe type changed. + if (serde_type_changed) { + _materialization_state.reset_dictionary(); + } + } + _decode_context.dictionary_index_only = is_dict_filter; + + _def_levels.clear(); + _rep_levels.clear(); + *read_rows = 0; + + if (_in_nested) { + return _read_nested_column(doris_column, materialization_type, filter_map, batch_size, + read_rows, eof, is_dict_filter); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + + do { + // generate the row ranges that should be read + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + // skip the whole page + _current_row_index = right_row; + } else { + bool skip_whole_batch = false; + // Determining whether to skip page or batch will increase the calculation time. + // When the filtering effect is greater than 60%, it is possible to skip the page or batch. + if (filter_map.has_filter() && filter_map.filter_ratio() > 0.6) { + // lazy read + size_t remaining_num_values = read_ranges.count(); + if (batch_size >= remaining_num_values && + filter_map.can_filter_all(remaining_num_values, _filter_map_index)) { + // We can skip the whole page if the remaining values are filtered by predicate columns + _filter_map_index += remaining_num_values; + _current_row_index = right_row; + *read_rows = remaining_num_values; + break; + } + skip_whole_batch = batch_size <= remaining_num_values && + filter_map.can_filter_all(batch_size, _filter_map_index); + if (skip_whole_batch) { + _filter_map_index += batch_size; + } + } + // load page data to decode or skip values + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); idx++) { + auto range = read_ranges.get_range(idx); + // generate the skipped values + size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + // generate the read values + size_t read_values = + std::min((size_t)(range.to() - range.from()), batch_size - has_read); + if (skip_whole_batch) { + RETURN_IF_ERROR(_skip_values(read_values)); + } else { + RETURN_IF_ERROR(_read_values(read_values, doris_column, materialization_type, + filter_map, is_dict_filter)); + } + has_read += read_values; + *read_rows += read_values; + _current_row_index += read_values; + if (has_read == batch_size) { + break; + } + } + } + } while (false); + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + + return Status::OK(); +} + +Status ArrayColumnReader::init(std::unique_ptr element_reader, + NativeFieldSchema* field) { + _field_schema = field; + _element_reader = std::move(element_reader); + return Status::OK(); +} + +Status ArrayColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_ARRAY) { + return Status::Corruption( + "Wrong data type for column '{}', expected Array type, actual type: {}.", + _field_schema->name, type->get_name()); + } + + ColumnPtr& element_column = assert_cast(*data_column).get_data_ptr(); + const DataTypePtr& element_type = + (assert_cast(remove_nullable(type).get()))->get_nested_type(); + // read nested column + RETURN_IF_ERROR(_element_reader->read_column_data(element_column, element_type, + root_node->element(), filter_map, batch_size, + read_rows, eof, is_dict_filter)); + if (*read_rows == 0) { + return Status::OK(); + } + + ColumnArray::Offsets64& offsets_data = assert_cast(*data_column).get_offsets(); + // fill offset and null map + RETURN_IF_ERROR(fill_array_offset(_field_schema, offsets_data, null_map_ptr, + _element_reader->get_rep_level(), + _element_reader->get_def_level())); + if (UNLIKELY(element_column->size() != offsets_data.back())) { + return Status::Corruption("Parquet array element count does not match repetition levels"); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::init(std::unique_ptr key_reader, + std::unique_ptr value_reader, NativeFieldSchema* field) { + _field_schema = field; + _key_reader = std::move(key_reader); + _value_reader = std::move(value_reader); + return Status::OK(); +} + +Status MapColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (remove_nullable(type)->get_primitive_type() != PrimitiveType::TYPE_MAP) { + return Status::Corruption( + "Wrong data type for column '{}', expected Map type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& map = assert_cast(*data_column); + const DataTypePtr& key_type = + assert_cast(remove_nullable(type).get())->get_key_type(); + const DataTypePtr& value_type = + assert_cast(remove_nullable(type).get())->get_value_type(); + ColumnPtr& key_column = map.get_keys_ptr(); + ColumnPtr& value_column = map.get_values_ptr(); + + size_t key_rows = 0; + size_t value_rows = 0; + bool key_eof = false; + bool value_eof = false; + int64_t orig_col_column_size = key_column->size(); + + RETURN_IF_ERROR(_key_reader->read_column_data(key_column, key_type, root_node->key(), + filter_map, batch_size, &key_rows, &key_eof, + is_dict_filter)); + + while (value_rows < key_rows && !value_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_value_reader->read_column_data( + value_column, value_type, root_node->value(), filter_map, key_rows - value_rows, + &loop_rows, &value_eof, is_dict_filter, key_column->size() - orig_col_column_size)); + value_rows += loop_rows; + } + if (UNLIKELY(key_rows != value_rows)) { + // MAP children share one logical-row boundary; EOF in one sibling is file corruption. + return Status::Corruption("Parquet map value reader returned {} rows for {} key rows", + value_rows, key_rows); + } + *read_rows = key_rows; + *eof = key_eof; + + if (*read_rows == 0) { + return Status::OK(); + } + + const auto parent_shape = [this](const ColumnReader& reader) { + std::vector> shape; + const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } + shape.reserve(rep_levels.size()); + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + // MAP siblings may have child-local collections and nullness. At this boundary, + // only the outer entry distribution and whether that entry exists must agree. + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); + } + } + return shape; + }; + if (UNLIKELY(_key_reader->get_rep_level().size() != _key_reader->get_def_level().size() || + _value_reader->get_rep_level().size() != _value_reader->get_def_level().size())) { + return Status::Corruption("Parquet map sibling level counts differ"); + } + if (UNLIKELY(parent_shape(*_key_reader) != parent_shape(*_value_reader))) { + return Status::Corruption("Parquet map key/value outer entry shapes differ"); + } + + if (UNLIKELY(key_column->size() != value_column->size())) { + return Status::Corruption("Parquet map key/value entry counts differ: {} vs {}", + key_column->size(), value_column->size()); + } + // Non-strict conversion can persist nullable Doris MAP keys, so the native reader must + // preserve writer output instead of reclassifying an otherwise valid file as corrupt. + const auto& key_rep_levels = _key_reader->get_rep_level(); + // fill offset and null map + // The key leaf is the canonical outer MAP shape. A nested value has additional repetition + // levels for its own collections, so comparing the two leaves would reject valid MAP values. + RETURN_IF_ERROR(fill_array_offset(_field_schema, map.get_offsets(), null_map_ptr, + key_rep_levels, _key_reader->get_def_level())); + if (UNLIKELY(key_column->size() != map.get_offsets().back())) { + return Status::Corruption("Parquet map entry count does not match repetition levels"); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + DORIS_CHECK(dynamic_cast(_key_reader.get()) == nullptr); + return _key_reader->read_column_levels(filter_map, batch_size, read_rows, eof); +} + +Status StructColumnReader::init( + std::unordered_map>&& child_readers, + NativeFieldSchema* field) { + _field_schema = field; + _child_readers = std::move(child_readers); + return Status::OK(); +} + +Status StructColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + _read_column_names.clear(); + for (const auto& child : _field_schema->children) { + auto reader = _child_readers.find(child.name); + DORIS_CHECK(reader != _child_readers.end()); + if (dynamic_cast(reader->second.get()) != nullptr) { + continue; + } + _read_column_names.emplace_back(child.name); + return reader->second->read_column_levels(filter_map, batch_size, read_rows, eof); + } + return Status::InternalError("Struct {} has no physical reader for levels", + _field_schema->name); +} +Status StructColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_STRUCT) { + return Status::Corruption( + "Wrong data type for column '{}', expected Struct type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& doris_struct = assert_cast(*data_column); + const auto* doris_struct_type = assert_cast(remove_nullable(type).get()); + + int64_t not_missing_column_id = -1; + size_t not_missing_orig_column_size = 0; + std::vector missing_column_idxs {}; + std::vector skip_reading_column_idxs {}; + std::vector> reference_parent_shape; + + auto parent_shape = [this](const ColumnReader& reader) { + std::vector> shape; + const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } + shape.reserve(rep_levels.size()); + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { + // Deeper repetitions belong to a nested child; only starts visible at this STRUCT's + // boundary and the optional parent's presence determine how siblings are paired. + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); + } + } + return shape; + }; + + _read_column_names.clear(); + + for (size_t i = 0; i < doris_struct.tuple_size(); ++i) { + ColumnPtr& doris_field = doris_struct.get_column_ptr(i); + auto& doris_type = doris_struct_type->get_element(i); + auto& doris_name = doris_struct_type->get_element_name(i); + if (!root_node->has_child(doris_name)) { + missing_column_idxs.push_back(i); + VLOG_DEBUG << "[ParquetReader] Missing column in schema: column_idx[" << i + << "], doris_name: " << doris_name << " (column not exists in root node)"; + continue; + } + auto file_name = root_node->file_child_name(doris_name); + + // Check if this is a SkipReadingReader - we should skip it when choosing reference column + // because SkipReadingReader doesn't know the actual data size in nested context + bool is_skip_reader = + dynamic_cast(_child_readers[file_name].get()) != nullptr; + + if (is_skip_reader) { + // Store SkipReadingReader columns to fill them later based on reference column size + skip_reading_column_idxs.push_back(i); + continue; + } + + // Only add non-SkipReadingReader columns to _read_column_names + // This ensures get_rep_level() and get_def_level() return valid levels + _read_column_names.emplace_back(file_name); + + size_t field_rows = 0; + bool field_eof = false; + if (not_missing_column_id == -1) { + not_missing_column_id = i; + not_missing_orig_column_size = doris_field->size(); + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, batch_size, + &field_rows, &field_eof, is_dict_filter)); + *read_rows = field_rows; + *eof = field_eof; + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } + reference_parent_shape = parent_shape(*_child_readers[file_name]); + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column + * (since `_read_nested_column` is not called repeatedly). + * + * It is worth mentioning that, theoretically, any sub-column can be chosen to fill the null_map, + * and selecting the shortest one will offer better performance + */ + } else { + while (field_rows < *read_rows && !field_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, + *read_rows - field_rows, &loop_rows, &field_eof, is_dict_filter)); + field_rows += loop_rows; + } + if (UNLIKELY(*read_rows != field_rows)) { + // STRUCT siblings must advance the same logical rows before any result is exposed. + return Status::Corruption("Parquet struct child '{}' returned {} rows, expected {}", + file_name, field_rows, *read_rows); + } + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } + if (UNLIKELY(parent_shape(*_child_readers[file_name]) != reference_parent_shape)) { + return Status::Corruption( + "Parquet struct child '{}' has a different repeated-parent shape", + file_name); + } + // DCHECK_EQ(*eof, field_eof); + } + } + + int64_t missing_column_sz = -1; + + if (not_missing_column_id == -1) { + // All queried columns are missing in the file (e.g., all added after schema change) + // We need to pick a column from _field_schema children that exists in the file for RL/DL reference + std::string reference_file_column_name; + std::unique_ptr* reference_reader = nullptr; + + for (const auto& child : _field_schema->children) { + auto it = _child_readers.find(child.name); + if (it != _child_readers.end()) { + // Skip SkipReadingReader as they don't have valid RL/DL + bool is_skip_reader = dynamic_cast(it->second.get()) != nullptr; + if (!is_skip_reader) { + reference_file_column_name = child.name; + reference_reader = &(it->second); + break; + } + } + } + + if (reference_reader != nullptr) { + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR( + (*reference_reader) + ->read_column_levels(filter_map, batch_size, &field_rows, &field_eof)); + + *read_rows = field_rows; + *eof = field_eof; + _read_column_names.emplace_back(reference_file_column_name); + missing_column_sz = 0; + const auto& rep_levels = (*reference_reader)->get_rep_level(); + const auto& def_levels = (*reference_reader)->get_def_level(); + DORIS_CHECK_EQ(rep_levels.size(), def_levels.size()); + for (size_t level_index = 0; level_index < def_levels.size(); ++level_index) { + if (def_levels[level_index] >= _field_schema->repeated_parent_def_level && + rep_levels[level_index] <= _field_schema->repetition_level) { + ++missing_column_sz; + } + } + } else { + return Status::Corruption( + "Cannot read struct '{}': all queried columns are missing and no reference " + "column found in file", + _field_schema->name); + } + } + + // This missing_column_sz is not *read_rows. Because read_rows returns the number of rows. + // For example: suppose we have a column array>, + // where b is a newly added column, that is, a missing column. + // There are two rows of data in this column, + // [{1,null},{2,null},{3,null}] + // [{4,null},{5,null}] + // When you first read subcolumn a, you read 5 data items and the value of *read_rows is 2. + // You should insert 5 records into subcolumn b instead of 2. + if (missing_column_sz == -1) { + missing_column_sz = doris_struct.get_column(not_missing_column_id).size() - + not_missing_orig_column_size; + } + + // Fill SkipReadingReader columns with the correct amount of data based on reference column + // Let SkipReadingReader handle the data filling through its read_column_data method + for (auto idx : skip_reading_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = const_cast(doris_struct_type->get_element(idx)); + auto& doris_name = const_cast(doris_struct_type->get_element_name(idx)); + auto file_name = root_node->file_child_name(doris_name); + + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->child(doris_name), filter_map, + missing_column_sz, &field_rows, &field_eof, is_dict_filter, missing_column_sz)); + } + + // Fill truly missing columns (not in root_node) with null or default value + for (auto idx : missing_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = doris_struct_type->get_element(idx); + DCHECK(doris_type->is_nullable()); + doris_field = IColumn::mutate(std::move(doris_field)); + auto mutable_column = doris_field->assert_mutable(); + auto* nullable_column = static_cast(mutable_column.get()); + nullable_column->insert_many_defaults(missing_column_sz); + } + + if (null_map_ptr != nullptr) { + fill_struct_null_map(_field_schema, *null_map_ptr, this->get_rep_level(), + this->get_def_level()); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h new file mode 100644 index 00000000000000..6ea2787f9f0211 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -0,0 +1,662 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "exprs/vexpr_fwd.h" +#include "format_v2/parquet/native_schema_node.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/common.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "storage/segment/row_ranges.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris::io { +struct IOContext; +} // namespace doris::io + +namespace doris::format::parquet::native { +using ::doris::ColumnString; +using segment_v2::RowRange; +using segment_v2::RowRanges; + +#ifdef BE_TEST +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context); +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode); +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row); +#endif + +class ColumnReader { +public: + struct ColumnStatistics { + ColumnStatistics() + : page_index_read_calls(0), + decompress_time(0), + decompress_cnt(0), + decode_header_time(0), + decode_value_time(0), + materialization_time(0), + hybrid_selection_batches(0), + hybrid_selection_ranges(0), + hybrid_selection_null_fallback_batches(0), + decode_dict_time(0), + decode_level_time(0), + decode_null_map_time(0), + skip_page_header_num(0), + parse_page_header_num(0), + read_page_header_time(0), + page_read_counter(0), + page_cache_write_counter(0), + page_cache_compressed_write_counter(0), + page_cache_decompressed_write_counter(0), + page_cache_hit_counter(0), + page_cache_missing_counter(0), + page_cache_compressed_hit_counter(0), + page_cache_decompressed_hit_counter(0) {} + + ColumnStatistics(ColumnChunkReaderStatistics& cs, int64_t null_map_time) + : page_index_read_calls(0), + decompress_time(cs.decompress_time), + decompress_cnt(cs.decompress_cnt), + decode_header_time(cs.decode_header_time), + decode_value_time(cs.decode_value_time), + materialization_time(cs.materialization_time), + hybrid_selection_batches(cs.hybrid_selection_batches), + hybrid_selection_ranges(cs.hybrid_selection_ranges), + hybrid_selection_null_fallback_batches(cs.hybrid_selection_null_fallback_batches), + decode_dict_time(cs.decode_dict_time), + decode_level_time(cs.decode_level_time), + decode_null_map_time(null_map_time), + skip_page_header_num(cs.skip_page_header_num), + parse_page_header_num(cs.parse_page_header_num), + read_page_header_time(cs.read_page_header_time), + page_read_counter(cs.page_read_counter), + page_cache_write_counter(cs.page_cache_write_counter), + page_cache_compressed_write_counter(cs.page_cache_compressed_write_counter), + page_cache_decompressed_write_counter(cs.page_cache_decompressed_write_counter), + page_cache_hit_counter(cs.page_cache_hit_counter), + page_cache_missing_counter(cs.page_cache_missing_counter), + page_cache_compressed_hit_counter(cs.page_cache_compressed_hit_counter), + page_cache_decompressed_hit_counter(cs.page_cache_decompressed_hit_counter), + leaf_page_read_counters {cs.data_page_read_counter} {} + + int64_t page_index_read_calls; + int64_t decompress_time; + int64_t decompress_cnt; + int64_t decode_header_time; + int64_t decode_value_time; + int64_t materialization_time; + int64_t hybrid_selection_batches; + int64_t hybrid_selection_ranges; + int64_t hybrid_selection_null_fallback_batches; + int64_t decode_dict_time; + int64_t decode_level_time; + int64_t decode_null_map_time; + int64_t skip_page_header_num; + int64_t parse_page_header_num; + int64_t read_page_header_time; + int64_t page_read_counter; + int64_t page_cache_write_counter; + int64_t page_cache_compressed_write_counter; + int64_t page_cache_decompressed_write_counter; + int64_t page_cache_hit_counter; + int64_t page_cache_missing_counter; + int64_t page_cache_compressed_hit_counter; + int64_t page_cache_decompressed_hit_counter; + // Preserve per-leaf identity when complex readers aggregate their other counters. + std::vector leaf_page_read_counters; + + void merge(ColumnStatistics& col_statistics) { + page_index_read_calls += col_statistics.page_index_read_calls; + decompress_time += col_statistics.decompress_time; + decompress_cnt += col_statistics.decompress_cnt; + decode_header_time += col_statistics.decode_header_time; + decode_value_time += col_statistics.decode_value_time; + materialization_time += col_statistics.materialization_time; + hybrid_selection_batches += col_statistics.hybrid_selection_batches; + hybrid_selection_ranges += col_statistics.hybrid_selection_ranges; + hybrid_selection_null_fallback_batches += + col_statistics.hybrid_selection_null_fallback_batches; + decode_dict_time += col_statistics.decode_dict_time; + decode_level_time += col_statistics.decode_level_time; + decode_null_map_time += col_statistics.decode_null_map_time; + skip_page_header_num += col_statistics.skip_page_header_num; + parse_page_header_num += col_statistics.parse_page_header_num; + read_page_header_time += col_statistics.read_page_header_time; + page_read_counter += col_statistics.page_read_counter; + leaf_page_read_counters.insert(leaf_page_read_counters.end(), + col_statistics.leaf_page_read_counters.begin(), + col_statistics.leaf_page_read_counters.end()); + page_cache_write_counter += col_statistics.page_cache_write_counter; + page_cache_compressed_write_counter += + col_statistics.page_cache_compressed_write_counter; + page_cache_decompressed_write_counter += + col_statistics.page_cache_decompressed_write_counter; + page_cache_hit_counter += col_statistics.page_cache_hit_counter; + page_cache_missing_counter += col_statistics.page_cache_missing_counter; + page_cache_compressed_hit_counter += col_statistics.page_cache_compressed_hit_counter; + page_cache_decompressed_hit_counter += + col_statistics.page_cache_decompressed_hit_counter; + } + }; + + ColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : _row_ranges(row_ranges), _total_rows(total_rows), _ctz(ctz), _io_ctx(io_ctx) {} + virtual ~ColumnReader() = default; + virtual Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter, + int64_t real_column_size = -1) = 0; + + // Evaluate a predicate scalar directly from a supported fixed-width page encoding. The default + // is a non-consuming fallback for nested and synthetic readers. + virtual Status read_fixed_width_filter(const VExprSPtrs&, int, FilterMap&, size_t, IColumn*, + IColumn::Filter* row_filter, size_t* read_rows, + bool* eof, bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + return Status::OK(); + } + + // Consume a nested batch while retaining only definition/repetition levels. This is used when + // schema evolution makes every projected STRUCT child synthetic: the parent still needs one + // physical leaf's shape, but decoding that leaf's strings or other payload would be wasted. + virtual Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) = 0; + + virtual Result materialize_dictionary_values(const ColumnInt32* dict_column, + const DataTypePtr& target_type) { + throw Exception( + Status::FatalError("Method materialize_dictionary_values is not supported")); + } + virtual Result dictionary_values(const DataTypePtr& target_type) { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); + } + + static Status create(io::FileReaderSPtr file, NativeFieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + const std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection = false, + const std::set& column_ids = {}, + const std::set& filter_column_ids = {}, + const std::string& page_cache_file_key = {}, + const ParquetReaderCompat& compat = {}, bool enable_strict_mode = false); + virtual const std::vector& get_rep_level() const = 0; + virtual const std::vector& get_def_level() const = 0; + virtual ColumnStatistics column_statistics() = 0; + virtual void close() = 0; + + // A repeated parent can expand one logical-row batch into millions of leaf values. Keep + // ordinary batch scratch for reuse, but let the top-level adapter release exceptional + // high-water allocations after every parent offset/null-map consumer has finished. + virtual void release_batch_scratch(size_t max_retained_bytes) = 0; + + virtual void reset_filter_map_index() = 0; + + NativeFieldSchema* get_field_schema() const { return _field_schema; } + void set_column_in_nested() { _in_nested = true; } + +protected: + void _generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const; + + NativeFieldSchema* _field_schema = nullptr; + const RowRanges& _row_ranges; + size_t _total_rows = 0; + const cctz::time_zone* _ctz = nullptr; + io::IOContext* _io_ctx = nullptr; + int64_t _current_row_index = 0; + int64_t _decode_null_map_time = 0; + + size_t _filter_map_index = 0; + std::set _filter_column_ids; + + // _in_nested: column in struct/map/array + // IN_COLLECTION : column in map/array + bool _in_nested = false; +}; + +template +class ScalarColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ScalarColumnReader) +public: + ScalarColumnReader(const RowRanges& row_ranges, size_t total_rows, + const tparquet::ColumnChunk& chunk_meta, + const tparquet::OffsetIndex* offset_index, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx), + _chunk_meta(chunk_meta), + _offset_index(offset_index) {} + ~ScalarColumnReader() override { close(); } + Status init(io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, + RuntimeState* state, const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_fixed_width_filter(const VExprSPtrs& conjuncts, int column_id, + FilterMap& filter_map, size_t batch_size, + IColumn* projected_column, IColumn::Filter* row_filter, + size_t* read_rows, bool* eof, bool* used_filter) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + Result materialize_dictionary_values(const ColumnInt32* dict_column, + const DataTypePtr& target_type) override; + Result dictionary_values(const DataTypePtr& target_type) override; + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { + return ColumnStatistics(_chunk_reader->chunk_statistics(), _decode_null_map_time); + } + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override; + +#ifdef BE_TEST + void reserve_batch_scratch_for_test(size_t elements); + size_t retained_batch_scratch_bytes_for_test() const; + size_t dictionary_materialization_count_for_test() const { + return _dictionary_materialization_count; + } +#endif + + void reset_filter_map_index() override { + _filter_map_index = 0; // nested + } + +private: + tparquet::ColumnChunk _chunk_meta; + const tparquet::OffsetIndex* _offset_index = nullptr; + std::unique_ptr _stream_reader; + std::unique_ptr> _chunk_reader; +#ifdef BE_TEST + size_t _dictionary_materialization_count = 0; +#endif + // rep def levels buffer. + std::vector _rep_levels; + std::vector _def_levels; + + size_t _current_range_idx = 0; + + Status gen_nested_null_map(size_t level_start_idx, size_t level_end_idx, + std::vector& null_map, + std::unordered_set& ancestor_null_indices) { + size_t has_read = level_start_idx; + null_map.emplace_back(0); + bool prev_is_null = false; + + while (has_read < level_end_idx) { + level_t def_level = _def_levels[has_read++]; + size_t loop_read = 1; + while (has_read < _def_levels.size() && _def_levels[has_read] == def_level) { + has_read++; + loop_read++; + } + + if (def_level < _field_schema->repeated_parent_def_level) { + for (size_t i = 0; i < loop_read; i++) { + ancestor_null_indices.insert(has_read - level_start_idx - loop_read + i); + } + continue; + } + + bool is_null = def_level < _field_schema->definition_level; + + if (prev_is_null == is_null && (USHRT_MAX - null_map.back() >= loop_read)) { + null_map.back() += loop_read; + } else { + if (!(prev_is_null ^ is_null)) { + null_map.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + null_map.emplace_back(USHRT_MAX); + null_map.emplace_back(0); + remaining -= USHRT_MAX; + } + null_map.emplace_back((u_short)remaining); + prev_is_null = is_null; + } + } + return Status::OK(); + } + + Status gen_filter_map(FilterMap& filter_map, size_t filter_loc, size_t level_start_idx, + size_t level_end_idx, std::vector& nested_filter_map_data, + FilterMap* nested_filter_map) { + DORIS_CHECK(nested_filter_map != nullptr); + nested_filter_map_data.resize(level_end_idx - level_start_idx); + for (size_t idx = level_start_idx; idx < level_end_idx; idx++) { + if (idx != level_start_idx && _rep_levels[idx] == 0) { + filter_loc++; + } + nested_filter_map_data[idx - level_start_idx] = + filter_map.filter_map_data()[filter_loc]; + } + + return nested_filter_map->init(nested_filter_map_data.data(), nested_filter_map_data.size(), + false); + } + + DataTypeSerDeSPtr _serde; + const IDataType* _serde_type = nullptr; + ParquetDecodeContext _decode_context; + ParquetMaterializationState _materialization_state; + bool _dictionary_index_only = false; + // Normal-size batch scratch is retained by the persistent leaf reader. Oversized allocations + // are released only after the top-level parent has consumed this leaf's level plan. + std::vector _null_run_lengths; + std::unordered_set _ancestor_null_indices; + std::vector _nested_filter_map_data; + NullMap _fixed_width_predicate_nulls; + IColumn::Filter _fixed_width_predicate_matches; + FilterMap _nested_filter_map; + ColumnSelectVector _select_vector; + uint8_t _oversized_scratch_idle_batches = 0; + + size_t retained_batch_scratch_bytes() const; + size_t active_batch_scratch_bytes() const; + + Status _skip_values(size_t num_values); + Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, bool is_dict_filter); + Status _read_fixed_width_filter_values(size_t num_values, const VExprSPtrs& conjuncts, + int column_id, FilterMap& filter_map, + IColumn* projected_column, IColumn::Filter* row_filter); + Status _read_nested_column(ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter); + Status _try_load_dict_page(bool* loaded, bool* has_dict); +}; + +class ArrayColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ArrayColumnReader) +public: + ArrayColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~ArrayColumnReader() override { close(); } + Status init(std::unique_ptr element_reader, NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override { + return _element_reader->read_column_levels(filter_map, batch_size, read_rows, eof); + } + const std::vector& get_rep_level() const override { + return _element_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _element_reader->get_def_level(); + } + ColumnStatistics column_statistics() override { return _element_reader->column_statistics(); } + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + _element_reader->release_batch_scratch(max_retained_bytes); + } + + void reset_filter_map_index() override { _element_reader->reset_filter_map_index(); } + +private: + std::unique_ptr _element_reader; +}; + +class MapColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(MapColumnReader) +public: + MapColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~MapColumnReader() override { close(); } + + Status init(std::unique_ptr key_reader, + std::unique_ptr value_reader, NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + return _key_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _key_reader->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics kst = _key_reader->column_statistics(); + ColumnStatistics vst = _value_reader->column_statistics(); + kst.merge(vst); + return kst; + } + + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + _key_reader->release_batch_scratch(max_retained_bytes); + _value_reader->release_batch_scratch(max_retained_bytes); + } + + void reset_filter_map_index() override { + _key_reader->reset_filter_map_index(); + _value_reader->reset_filter_map_index(); + } + +private: + std::unique_ptr _key_reader; + std::unique_ptr _value_reader; +}; + +class StructColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(StructColumnReader) +public: + StructColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~StructColumnReader() override { close(); } + + Status init(std::unordered_map>&& child_readers, + NativeFieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + if (!_read_column_names.empty()) { + // can't use _child_readers[*_read_column_names.begin()] + // because the operator[] of std::unordered_map is not const :( + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column, + * that is `_read_column_names.front()`. + */ + return _child_readers.find(_read_column_names.front())->second->get_rep_level(); + } + return _child_readers.begin()->second->get_rep_level(); + } + + const std::vector& get_def_level() const override { + if (!_read_column_names.empty()) { + return _child_readers.find(_read_column_names.front())->second->get_def_level(); + } + return _child_readers.begin()->second->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics st; + for (const auto& column_name : _read_column_names) { + auto reader = _child_readers.find(column_name); + if (reader != _child_readers.end()) { + ColumnStatistics cst = reader->second->column_statistics(); + st.merge(cst); + } + } + return st; + } + + void close() override {} + + void release_batch_scratch(size_t max_retained_bytes) override { + for (const auto& reader : _child_readers) { + reader.second->release_batch_scratch(max_retained_bytes); + } + } + + void reset_filter_map_index() override { + for (const auto& reader : _child_readers) { + reader.second->reset_filter_map_index(); + } + } + +private: + std::unordered_map> _child_readers; + std::vector _read_column_names; + //Need to use vector instead of set,see `get_rep_level()` for the reason. +}; + +// A special reader that skips actual reading but provides empty data with correct structure +// This is used when a column is not needed but its structure is required (e.g., for map keys) +class SkipReadingReader : public ColumnReader { +public: + SkipReadingReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx, NativeFieldSchema* field_schema) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) { + _field_schema = field_schema; // Use inherited member from base class + VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: " + << _field_schema->name; + } + + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override { + VLOG_DEBUG << "[ParquetReader] SkipReadingReader::read_column_data for field: " + << _field_schema->name << ", batch_size: " << batch_size; + DCHECK(real_column_size >= 0); // real_column_size for filtered column size. + + // Simulate reading without actually reading data + // Fill with default/null values based on column type + doris_column = IColumn::mutate(std::move(doris_column)); + MutableColumnPtr data_column = doris_column->assert_mutable(); + + if (real_column_size > 0) { + if (is_column_nullable(*doris_column)) { + auto* nullable_column = static_cast(data_column.get()); + nullable_column->insert_many_defaults(real_column_size); + } else { + // For non-nullable columns, insert appropriate default values + for (size_t i = 0; i < real_column_size; ++i) { + data_column->insert_default(); + } + } + } + + *read_rows = batch_size; // Indicate we "read" batch_size rows + *eof = false; // We can always provide more empty data + + VLOG_DEBUG << "[ParquetReader] SkipReadingReader generated " << batch_size + << " default values for field: " << _field_schema->name; + + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t*, bool*) override { + return Status::InternalError("Skip reader cannot provide Parquet levels for field {}", + _field_schema->name); + } + + static std::unique_ptr create_unique(const RowRanges& row_ranges, + size_t total_rows, cctz::time_zone* ctz, + io::IOContext* io_ctx, + NativeFieldSchema* field_schema) { + return std::make_unique(row_ranges, total_rows, ctz, io_ctx, + field_schema); + } + + // These methods should not be called for SkipReadingReader + // If they are called, it indicates a logic error in the code + const std::vector& get_rep_level() const override { + LOG(FATAL) << "get_rep_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + const std::vector& get_def_level() const override { + LOG(FATAL) << "get_def_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + // Implement required pure virtual methods from base class + ColumnStatistics column_statistics() override { + return ColumnStatistics(); // Return empty statistics + } + + void close() override { + // Nothing to close for skip reading + } + + void release_batch_scratch(size_t) override {} + + void reset_filter_map_index() override { _filter_map_index = 0; } +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/common.cpp b/be/src/format_v2/parquet/reader/native/common.cpp new file mode 100644 index 00000000000000..f048d2b878ebee --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.cpp @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/common.h" + +#include + +#include "core/types.h" +#include "util/simd/bits.h" + +namespace doris::format::parquet::native { + +Status FilterMap::init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all) { + _filter_all = filter_all; + _filter_map_data = filter_map_data; + _filter_map_size = filter_map_size; + if (filter_all) { + _has_filter = true; + _filter_ratio = 1; + } else if (filter_map_data == nullptr) { + _has_filter = false; + _filter_ratio = 0; + } else { + const size_t filter_count = simd::count_zero_num( + reinterpret_cast(filter_map_data), filter_map_size); + if (filter_count == filter_map_size) { + _has_filter = true; + _filter_all = true; + _filter_ratio = 1; + } else if (filter_count > 0 && filter_map_size > 0) { + _has_filter = true; + _filter_ratio = static_cast(filter_count) / filter_map_size; + } else { + _has_filter = false; + _filter_ratio = 0; + } + } + return Status::OK(); +} + +bool FilterMap::can_filter_all(size_t remaining_num_values, size_t filter_map_index) { + if (!_has_filter) { + return false; + } + if (_filter_all) { + DCHECK_LE(remaining_num_values + filter_map_index, _filter_map_size); + return true; + } + if (remaining_num_values + filter_map_index > _filter_map_size) { + return false; + } + return simd::count_zero_num( + reinterpret_cast(_filter_map_data + filter_map_index), + remaining_num_values) == remaining_num_values; +} + +Status FilterMap::generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index) const { + if (!has_filter() || filter_all()) { + return Status::InternalError( + "Native FilterMap requires a partial filter: has_filter={}, filter_all={}", + has_filter(), filter_all()); + } + if (rep_levels.empty()) { + return Status::OK(); + } + + nested_filter_map_data.resize(rep_levels.size()); + size_t current_row = current_row_ptr != nullptr ? *current_row_ptr : 0; + for (size_t i = start_index; i < rep_levels.size(); ++i) { + if (i != start_index && rep_levels[i] == 0) { + ++current_row; + if (current_row >= _filter_map_size) { + return Status::InvalidArgument("Nested filter row {} exceeds filter map size {}", + current_row, _filter_map_size); + } + } + nested_filter_map_data[i] = _filter_map_data[current_row]; + } + if (current_row_ptr != nullptr) { + *current_row_ptr = current_row; + } + + auto new_filter = std::make_unique(); + RETURN_IF_ERROR( + new_filter->init(nested_filter_map_data.data(), nested_filter_map_data.size(), false)); + *nested_filter_map = std::move(new_filter); + return Status::OK(); +} + +Status ColumnSelectVector::init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices) { + _num_values = num_values; + _num_nulls = 0; + _read_index = 0; + size_t map_index = 0; + bool is_null = false; + _has_filter = filter_map->has_filter(); + + if (_has_filter) { + _data_map.resize(num_values); + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_NULL; + } + } else { + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_CONTENT; + } + } + is_null = !is_null; + } + + size_t num_read = 0; + size_t source_index = 0; + size_t valid_count = 0; + while (valid_count < num_values) { + DCHECK_LT(filter_map_index + source_index, filter_map->filter_map_size()); + if (skipped_indices != nullptr && + skipped_indices->contains(filter_map_index + source_index)) { + ++source_index; + continue; + } + if (filter_map->filter_map_data()[filter_map_index + source_index] != 0) { + _data_map[valid_count] = + _data_map[valid_count] == FILTERED_NULL ? NULL_DATA : CONTENT; + ++num_read; + } + ++valid_count; + ++source_index; + } + _num_filtered = num_values - num_read; + + if (null_map != nullptr && num_read > 0) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_read); + if (_num_nulls == 0) { + memset(null_map->data() + null_map_index, 0, num_read); + } else if (_num_nulls == num_values) { + memset(null_map->data() + null_map_index, 1, num_read); + } else { + for (size_t i = 0; i < num_values; ++i) { + if (_data_map[i] == CONTENT) { + (*null_map)[null_map_index++] = static_cast(false); + } else if (_data_map[i] == NULL_DATA) { + (*null_map)[null_map_index++] = static_cast(true); + } + } + } + } + } else { + _num_filtered = 0; + _run_length_null_map = &run_length_null_map; + if (null_map != nullptr) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_values); + for (const auto run_length : run_length_null_map) { + memset(null_map->data() + null_map_index, is_null ? 1 : 0, run_length); + null_map_index += run_length; + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } else { + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/common.h b/be/src/format_v2/parquet/reader/native/common.h new file mode 100644 index 00000000000000..4ad204da2ff30d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_nullable.h" + +namespace doris::format::parquet::native { + +// These cursor types belong to the native v2 decoder so its execution path never reaches into +// the legacy Parquet reader for filtering or level materialization. +using level_t = int16_t; + +class FilterMap { +public: + Status init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all); + + Status generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index = 0) const; + + const uint8_t* filter_map_data() const { return _filter_map_data; } + size_t filter_map_size() const { return _filter_map_size; } + bool has_filter() const { return _has_filter; } + bool filter_all() const { return _filter_all; } + double filter_ratio() const { return _has_filter ? _filter_ratio : 0; } + + bool can_filter_all(size_t remaining_num_values, size_t filter_map_index); + +private: + bool _filter_all = false; + bool _has_filter = false; + const uint8_t* _filter_map_data = nullptr; + size_t _filter_map_size = 0; + double _filter_ratio = 0; +}; + +class ColumnSelectVector { +public: + enum DataReadType : uint8_t { CONTENT = 0, NULL_DATA, FILTERED_CONTENT, FILTERED_NULL }; + + Status init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices = nullptr); + + size_t num_values() const { return _num_values; } + size_t num_nulls() const { return _num_nulls; } + size_t num_filtered() const { return _num_filtered; } + bool has_filter() const { return _has_filter; } + + template + size_t get_next_run(DataReadType* data_read_type) { + DCHECK_EQ(_has_filter, has_filter); + if constexpr (has_filter) { + if (_read_index == _num_values) { + return 0; + } + const DataReadType type = _data_map[_read_index++]; + size_t run_length = 1; + while (_read_index < _num_values && _data_map[_read_index] == type) { + ++run_length; + ++_read_index; + } + *data_read_type = type; + return run_length; + } + + size_t run_length = 0; + while (run_length == 0) { + if (_read_index == _run_length_null_map->size()) { + return 0; + } + *data_read_type = _read_index % 2 == 0 ? CONTENT : NULL_DATA; + run_length = (*_run_length_null_map)[_read_index++]; + } + return run_length; + } + +private: + std::vector _data_map; + const std::vector* _run_length_null_map = nullptr; + bool _has_filter = false; + size_t _num_values = 0; + size_t _num_nulls = 0; + size_t _num_filtered = 0; + size_t _read_index = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/decoder.cpp b/be/src/format_v2/parquet/reader/native/decoder.cpp new file mode 100644 index 00000000000000..5f6a978317bde7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.cpp @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/decoder.h" + +#include +#include + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" +#include "format_v2/parquet/reader/native/fix_length_dict_decoder.hpp" +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +namespace doris::format::parquet::native { +namespace { +Status unsupported_type(tparquet::Type::type type, tparquet::Encoding::type encoding) { + return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", + tparquet::to_string(type), tparquet::to_string(encoding)); +} + +Status create_plain_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::PLAIN); + } +} + +Status create_dictionary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + return Status::InternalError("Boolean type cannot have a dictionary page"); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT96: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FLOAT: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::DOUBLE: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique>(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::RLE_DICTIONARY); + } +} + +Status create_delta_binary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + default: + return Status::InternalError("DELTA_BINARY_PACKED only supports INT32 and INT64"); + } +} + +Status create_byte_stream_split_decoder(tparquet::Type::type type, + std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::BYTE_STREAM_SPLIT); + } +} +} // namespace + +Status Decoder::get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder) { + switch (encoding) { + case tparquet::Encoding::PLAIN: + return create_plain_decoder(type, decoder); + case tparquet::Encoding::RLE_DICTIONARY: + return create_dictionary_decoder(type, decoder); + case tparquet::Encoding::RLE: + if (type != tparquet::Type::BOOLEAN) { + return unsupported_type(type, encoding); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::DELTA_BINARY_PACKED: + return create_delta_binary_decoder(type, decoder); + case tparquet::Encoding::DELTA_BYTE_ARRAY: + if (type != tparquet::Type::BYTE_ARRAY && type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) { + return Status::InternalError( + "DELTA_BYTE_ARRAY only supports BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY."); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + if (type != tparquet::Type::BYTE_ARRAY) { + return Status::InternalError("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY."); + } + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Encoding::BYTE_STREAM_SPLIT: + return create_byte_stream_split_decoder(type, decoder); + default: + return Status::InternalError("Unsupported encoding {}(type={}) in parquet decoder", + tparquet::to_string(encoding), tparquet::to_string(type)); + } +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h new file mode 100644 index 00000000000000..379176d5e459e0 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __AVX2__ +#include +#endif + +#include "common/status.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/types.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { + +inline bool dictionary_indices_in_bounds(const uint32_t* indices, size_t count, + size_t dictionary_size) { + if (count == 0) { + return true; + } + if (dictionary_size == 0) { + return false; + } + uint32_t max_index = 0; + size_t row = 0; +#ifdef __AVX2__ + __m256i vector_max = _mm256_setzero_si256(); + for (; row + 8 <= count; row += 8) { + const auto values = _mm256_loadu_si256(reinterpret_cast(indices + row)); + vector_max = _mm256_max_epu32(vector_max, values); + } + alignas(32) uint32_t lanes[8]; + _mm256_store_si256(reinterpret_cast<__m256i*>(lanes), vector_max); + for (const auto lane : lanes) { + max_index = std::max(max_index, lane); + } +#endif + for (; row < count; ++row) { + max_index = std::max(max_index, indices[row]); + } + return static_cast(max_index) < dictionary_size; +} + +class Decoder : public ParquetDecodeSource { +public: + Decoder() = default; + virtual ~Decoder() = default; + + static Status get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder); + + // The type with fix length + void set_type_length(int32_t type_length) { _type_length = type_length; } + + // Set the data to be decoded + virtual Status set_data(Slice* data) { + _data = data; + _offset = 0; + return Status::OK(); + } + + // Page headers provide an upper bound before encoding-specific headers are trusted. + virtual void set_expected_values(size_t expected_values) { _expected_values = expected_values; } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + return Status::NotSupported("Fixed values are not supported by this Parquet decoder"); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + return Status::NotSupported("Binary values are not supported by this Parquet decoder"); + } + + Status skip_values(size_t num_values) override = 0; + + virtual void release_scratch(size_t max_retained_bytes) {} + virtual size_t retained_scratch_bytes() const { return 0; } + virtual size_t active_scratch_bytes() const { return 0; } + + virtual Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + return Status::NotSupported("set_dict is not supported"); + } + +protected: + template + static void release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + if (values->capacity() * sizeof(T) > max_retained_bytes) { + std::vector().swap(*values); + } + } + + int32_t _type_length = -1; + Slice* _data = nullptr; + // Page offsets are host-sized so checked advances cannot wrap at the 4 GiB boundary. + size_t _offset = 0; + size_t _expected_values = std::numeric_limits::max(); +}; + +class BaseDictDecoder : public Decoder { +public: + BaseDictDecoder() = default; + ~BaseDictDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + if (UNLIKELY(data == nullptr || data->size == 0)) { + return Status::Corruption("Parquet dictionary index stream is empty"); + } + _data = data; + _offset = 0; + uint8_t bit_width = *data->data; + // Dictionary indices are uint32_t; wider external widths make repeated runs overwrite the + // decoder's four-byte state before any dictionary-bound check can run. + if (UNLIKELY(bit_width > 32)) { + return Status::Corruption("Parquet dictionary index bit width {} exceeds 32", + bit_width); + } + _index_batch_decoder = std::make_unique>( + reinterpret_cast(data->data) + 1, static_cast(data->size) - 1, + bit_width); + return Status::OK(); + } + + bool has_dictionary() const override { return true; } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + indices->resize(num_values); + const auto decoded = + _index_batch_decoder->GetBatch(indices->data(), cast_set(num_values)); + if (UNLIKELY(decoded != num_values)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t num_dictionary_values = dictionary_size(); + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data(), num_values, + num_dictionary_values))) { + // The SIMD common path only computes a bound; recover the exact corrupt row for the + // diagnostic after the batch has already been proven invalid. + for (size_t row = 0; row < num_values; ++row) { + if ((*indices)[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[row], row, num_dictionary_values); + } + } + return Status::OK(); + } + + Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + const size_t num_dictionary_values = dictionary_size(); + indices->resize(selection.selected_values); + size_t cursor = 0; + size_t output = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + const auto decoded = _index_batch_decoder->GetBatch(indices->data() + output, + cast_set(range.count)); + if (UNLIKELY(decoded != range.count)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data() + output, range.count, + num_dictionary_values))) { + for (size_t row = 0; row < range.count; ++row) { + if ((*indices)[output + row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[output + row], range.first + row, num_dictionary_values); + } + } + output += range.count; + cursor = range.first + range.count; + } + DORIS_CHECK(cursor <= selection.total_values); + RETURN_IF_ERROR(_decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values)); + DORIS_CHECK_EQ(output, selection.selected_values); + return Status::OK(); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + return _decode_dictionary_values(num_values, 0, dictionary_size(), consumer); + } + + Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) override { + const size_t num_dictionary_values = dictionary_size(); + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + RETURN_IF_ERROR(_decode_dictionary_values(range.count, range.first, + num_dictionary_values, consumer)); + cursor = range.first + range.count; + } + DORIS_CHECK(cursor <= selection.total_values); + return _decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values); + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_skip_indices, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _skip_indices.capacity() * sizeof(uint32_t); + } + size_t active_scratch_bytes() const override { return _skip_indices.size() * sizeof(uint32_t); } + +protected: + Status skip_values(size_t num_values) override { + return _decode_and_validate_skipped(num_values, 0, dictionary_size()); + } + + Status _decode_and_validate_skipped(size_t num_values, size_t row_offset, + size_t num_dictionary_values) { + constexpr size_t kSkipBatchSize = 4096; + // Skipped dictionary ids are still external input and must be bounds-checked, but keeping + // only one bounded gap buffer avoids the page-sized scratch used by sparse selections. + _skip_indices.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped_values = 0; + while (skipped_values < num_values) { + const size_t batch_size = std::min(num_values - skipped_values, kSkipBatchSize); + const auto skipped = _index_batch_decoder->GetBatch(_skip_indices.data(), + static_cast(batch_size)); + if (UNLIKELY(skipped != batch_size)) { + return Status::IOError( + "Can't skip enough Parquet dictionary indices at row {}: {} of {}", + row_offset + skipped_values, skipped, batch_size); + } + // Filter gaps may be huge RLE runs; validate them in bounded SIMD-sized batches. + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch_size, + num_dictionary_values))) { + for (size_t row = 0; row < batch_size; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at skipped row {} exceeds dictionary " + "size {}", + _skip_indices[row], row_offset + skipped_values + row, + num_dictionary_values); + } + } + skipped_values += batch_size; + } + return Status::OK(); + } + + Status _decode_dictionary_values(size_t num_values, size_t row_offset, + size_t num_dictionary_values, + ParquetDictionaryValueConsumer& consumer) { + constexpr size_t kLiteralBatchSize = 1024; + size_t decoded_values = 0; + while (decoded_values < num_values) { + const int32_t repeats = _index_batch_decoder->NextNumRepeats(); + if (repeats > 0) { + const size_t run = std::min(repeats, num_values - decoded_values); + const uint32_t index = + _index_batch_decoder->GetRepeatedValue(cast_set(run)); + if (UNLIKELY(static_cast(index) >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + index, row_offset + decoded_values, num_dictionary_values); + } + RETURN_IF_ERROR(consumer.consume_repeated(index, run)); + decoded_values += run; + continue; + } + + const int32_t literals = _index_batch_decoder->NextNumLiterals(); + if (UNLIKELY(literals == 0)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t batch = std::min({static_cast(literals), + num_values - decoded_values, kLiteralBatchSize}); + _skip_indices.resize(batch); + if (UNLIKELY(!_index_batch_decoder->GetLiteralValues(cast_set(batch), + _skip_indices.data()))) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch, + num_dictionary_values))) { + for (size_t row = 0; row < batch; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + _skip_indices[row], row_offset + decoded_values + row, + num_dictionary_values); + } + } + RETURN_IF_ERROR(consumer.consume_indices(_skip_indices.data(), batch)); + decoded_values += batch; + } + return Status::OK(); + } + + // For dictionary encoding + DorisUniqueBufferPtr _dict; + std::unique_ptr> _index_batch_decoder; + std::vector _skip_indices; + uint64_t _dictionary_generation = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp new file mode 100644 index 00000000000000..c94aead8caa6b6 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" + +namespace doris::format::parquet::native { +Status DeltaLengthByteArrayDecoder::_init_lengths() { + auto length_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_len_decoder.set_bit_reader(length_reader)); + const uint32_t num_lengths = _len_decoder.valid_values_count(); + + DeltaBitPackDecoder length_locator; + length_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(length_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector lengths(std::min(num_lengths, kLocateBatchSize)); + size_t located = 0; + int64_t payload_size = 0; + while (located < num_lengths) { + const size_t batch_size = std::min(num_lengths - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR( + length_locator.decode(lengths.data(), static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + for (size_t i = 0; i < batch_size; ++i) { + if (UNLIKELY(lengths[i] < 0) || + common::add_overflow(payload_size, static_cast(lengths[i]), + payload_size)) { + return Status::Corruption("Invalid Parquet delta byte-array length"); + } + } + located += batch_size; + } + // The locator leaves the shared reader at the first payload byte without retaining every + // length; the independent decoder supplies bounded batches during materialization or skip. + if (UNLIKELY(payload_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + payload_size, _bit_reader->bytes_left()); + } + _num_valid_values = num_lengths; + return Status::OK(); +} + +Status DeltaLengthByteArrayDecoder::_get_internal(Slice* buffer, int max_values, + int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + + int64_t data_size = 0; + _buffered_length.resize(max_values); + uint32_t lengths_decoded = 0; + RETURN_IF_ERROR(_len_decoder.decode(_buffered_length.data(), static_cast(max_values), + &lengths_decoded)); + if (UNLIKELY(lengths_decoded != max_values)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + const int32_t* length_ptr = _buffered_length.data(); + for (int i = 0; i < max_values; ++i) { + int32_t len = length_ptr[i]; + if (len < 0) [[unlikely]] { + return Status::InvalidArgument("Negative string delta length"); + } + buffer[i].size = len; + if (common::add_overflow(data_size, static_cast(len), data_size)) { + return Status::InvalidArgument("Excess expansion in DELTA_(LENGTH_)BYTE_ARRAY"); + } + } + // Every declared byte must exist in this page. Check before resize so a tiny malformed stream + // cannot reserve memory based only on attacker-controlled decoded lengths. + if (UNLIKELY(data_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + data_size, _bit_reader->bytes_left()); + } + _buffered_data.resize(data_size); + char* data_ptr = _buffered_data.data(); + for (int64_t j = 0; j < data_size; j++) { + if (!_bit_reader->GetValue(8, data_ptr + j)) { + return Status::IOError("Get length bytes EOF"); + } + } + + for (int i = 0; i < max_values; ++i) { + buffer[i].data = data_ptr; + data_ptr += buffer[i].size; + } + // this->num_values_ -= max_values; + _num_valid_values -= max_values; + *out_num_values = max_values; + return Status::OK(); +} + +Status DeltaByteArrayDecoder::_get_internal(Slice* buffer, int max_values, int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = max_values; + return Status::OK(); + } + + int suffix_read; + RETURN_IF_ERROR(_suffix_decoder.decode(buffer, max_values, &suffix_read)); + if (suffix_read != max_values) [[unlikely]] { + return Status::IOError("Read {}, expecting {} from suffix decoder", + std::to_string(suffix_read), std::to_string(max_values)); + } + + int64_t data_size = 0; + _buffered_prefix_length.resize(max_values); + uint32_t prefixes_decoded = 0; + RETURN_IF_ERROR(_prefix_len_decoder.decode( + _buffered_prefix_length.data(), static_cast(max_values), &prefixes_decoded)); + if (UNLIKELY(prefixes_decoded != max_values)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + const int32_t* prefix_len_ptr = _buffered_prefix_length.data(); + size_t preceding_value_size = _last_value.size(); + for (int i = 0; i < max_values; ++i) { + if (prefix_len_ptr[i] < 0) [[unlikely]] { + return Status::InvalidArgument("negative prefix length in DELTA_BYTE_ARRAY"); + } + const size_t prefix_size = static_cast(prefix_len_ptr[i]); + if (prefix_size > preceding_value_size) [[unlikely]] { + // Prefixes form a dependency chain, so validate each one before aggregate allocation. + return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); + } + size_t reconstructed_size = 0; + if (common::add_overflow(prefix_size, buffer[i].size, reconstructed_size) || + reconstructed_size > static_cast(std::numeric_limits::max()) || + common::add_overflow(data_size, static_cast(reconstructed_size), data_size)) + [[unlikely]] { + return Status::InvalidArgument("excess expansion in DELTA_BYTE_ARRAY"); + } + preceding_value_size = reconstructed_size; + } + _buffered_data.resize(data_size); + + std::string_view prefix {_last_value}; + + char* data_ptr = _buffered_data.data(); + for (int i = 0; i < max_values; ++i) { + memcpy(data_ptr, prefix.data(), prefix_len_ptr[i]); + // buffer[i] currently points to the string suffix + memcpy(data_ptr + prefix_len_ptr[i], buffer[i].data, buffer[i].size); + buffer[i].data = data_ptr; + buffer[i].size += prefix_len_ptr[i]; + data_ptr += buffer[i].size; + prefix = std::string_view {buffer[i].data, buffer[i].size}; + } + _num_valid_values -= max_values; + _last_value = std::string {prefix}; + + if (_num_valid_values == 0) { + _last_value_in_previous_page = _last_value; + } + *out_num_values = max_values; + return Status::OK(); +} +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h new file mode 100644 index 00000000000000..db9efecaced470 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -0,0 +1,818 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "exec/common/arithmetic_overflow.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +namespace detail { + +inline Status checked_delta_padding_bits(uint32_t bit_width, uint32_t remaining_values, + int64_t* padding_bits) { + DORIS_CHECK(padding_bits != nullptr); + const uint64_t widened_bits = static_cast(bit_width) * remaining_values; + if (UNLIKELY(widened_bits > static_cast(std::numeric_limits::max()))) { + return Status::Corruption("Parquet delta miniblock padding is too large"); + } + *padding_bits = static_cast(widened_bits); + return Status::OK(); +} + +} // namespace detail + +class DeltaDecoder : public Decoder { +public: + DeltaDecoder() = default; + ~DeltaDecoder() override = default; +}; + +/** + * Format + * [header] [block 1] [block 2] ... [block N] + * Header + * [block size] [_mini_blocks_per_block] [_total_value_count] [first value] + * Block + * [min delta] [list of bitwidths of the mini blocks] [miniblocks] + */ +template +class DeltaBitPackDecoder final : public DeltaDecoder { +public: + using UT = std::make_unsigned_t; + + DeltaBitPackDecoder() = default; + ~DeltaBitPackDecoder() override = default; + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + uint32_t num_valid_values = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), static_cast(batch_size), + &num_valid_values)); + // Skips retain exact validation without allocating in proportion to a sparse gap. + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError("Expected to skip {} Parquet delta values, skipped {}", + num_values, skipped + num_valid_values); + } + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + _values.resize(num_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", num_values, + decoded_count); + } + return consumer.consume(reinterpret_cast(_values.data()), num_values, + sizeof(T)); + } + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + _values.resize(selection.total_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", + selection.total_values, decoded_count); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, range.count * sizeof(T)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(reinterpret_cast(_values.data()), output, + sizeof(T)); + } + + Status decode(T* buffer, uint32_t num_values, uint32_t* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + uint32_t valid_values_count() { + // _total_value_count in header ignores of null values + return _total_values_remaining; + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + // The bit-width table drives later miniblock transitions, so it is reclaimable only after + // the page is exhausted; _values contains completed batch output and is always disposable. + if (_total_values_remaining == 0) { + release_vector_if_oversized(&_delta_bit_widths, max_retained_bytes); + } + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(T) + _delta_bit_widths.capacity() * sizeof(uint8_t); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(T) + _delta_bit_widths.size() * sizeof(uint8_t); + } + + Status set_data(Slice* slice) override { + _bit_reader.reset( + new BitReader((const uint8_t*)slice->data, cast_set(slice->size))); + RETURN_IF_ERROR(_init_header()); + _data = slice; + _offset = 0; + return Status::OK(); + } + + // Set BitReader which is already initialized by DeltaLengthByteArrayDecoder or + // DeltaByteArrayDecoder + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_init_header()); + return Status::OK(); + } + +private: + static constexpr int kMaxDeltaBitWidth = static_cast(sizeof(T) * 8); + Status _init_header(); + Status _init_block(); + Status _init_mini_block(int bit_width); + Status _get_internal(T* buffer, uint32_t max_values, uint32_t* out_num_values); + + std::vector _values; + + std::shared_ptr _bit_reader; + uint32_t _values_per_block; + uint32_t _mini_blocks_per_block; + uint32_t _values_per_mini_block; + uint32_t _total_value_count; + + T _min_delta; + T _last_value; + + uint32_t _mini_block_idx; + std::vector _delta_bit_widths; + int _delta_bit_width; + // If the page doesn't contain any block, `_block_initialized` will + // always be false. Otherwise, it will be true when first block initialized. + bool _block_initialized; + + uint32_t _total_values_remaining; + // Remaining values in current mini block. If the current block is the last mini block, + // _values_remaining_current_mini_block may greater than _total_values_remaining. + uint32_t _values_remaining_current_mini_block; +}; + +class DeltaLengthByteArrayDecoder final : public DeltaDecoder { +public: + explicit DeltaLengthByteArrayDecoder() + : _len_decoder(), _buffered_length(0), _buffered_data(0) {} + + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _len_decoder.set_expected_values(expected_values); + } + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + int num_valid_values = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), static_cast(batch_size), &num_valid_values)); + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError( + "Expected to skip {} Parquet delta-length values, skipped {}", num_values, + skipped + num_valid_values); + } + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + num_values, decoded_count); + } + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + _values.resize(selection.total_values); + int decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + selection.total_values, decoded_count); + } + _string_refs.resize(selection.selected_values); + size_t output = 0; + for (const auto& range : selection.ranges) { + for (size_t row = 0; row < range.count; ++row) { + const auto& value = _values[range.first + row]; + _string_refs[output++] = StringRef(value.data, value.size); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + int valid_values_count() const { return _num_valid_values; } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_buffered_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _len_decoder.release_scratch(max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _buffered_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _len_decoder.retained_scratch_bytes(); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _buffered_length.size() * sizeof(int32_t) + _buffered_data.size() * sizeof(char) + + _len_decoder.active_scratch_bytes(); + } + + Status set_data(Slice* slice) override { + if (slice == nullptr || slice->size == 0) { + // Reused decoders must never retain lengths or payload pointers from the prior page. + _bit_reader.reset(); + _data = nullptr; + _offset = 0; + _num_valid_values = 0; + _buffered_length.clear(); + _buffered_data.clear(); + return Status::Corruption("Parquet delta-length page is empty"); + } + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + _data = slice; + _offset = 0; + RETURN_IF_ERROR(_init_lengths()); + return Status::OK(); + } + + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_init_lengths()); + return Status::OK(); + } + +private: + Status _init_lengths(); + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _len_decoder; + + int _num_valid_values; + std::vector _buffered_length; + std::vector _buffered_data; +}; + +class DeltaByteArrayDecoder : public DeltaDecoder { +public: + explicit DeltaByteArrayDecoder() : _buffered_prefix_length(0), _buffered_data(0) {} + + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _prefix_len_decoder.set_expected_values(expected_values); + _suffix_decoder.set_expected_values(expected_values); + } + + Status skip_values(size_t num_values) override { + constexpr size_t kSkipBatchSize = 4096; + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + RETURN_IF_ERROR(_skip_slices(batch_size)); + skipped += batch_size; + } + return Status::OK(); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + _selected_binary_values.clear(); + _selected_value_sizes.clear(); + _selected_value_sizes.reserve(selection.selected_values); + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + size_t decoded = 0; + while (decoded < range.count) { + const size_t batch_size = _bounded_reconstruction_batch_size(range.count - decoded); + RETURN_IF_ERROR(_decode_slices(batch_size)); + for (const auto& value : _values) { + if (UNLIKELY(value.size > std::numeric_limits::max() - + _selected_binary_values.size())) { + return Status::IOError( + "Parquet delta-byte-array selection byte size overflows"); + } + if (value.size > 0) { + _selected_binary_values.insert(_selected_binary_values.end(), value.data, + value.data + value.size); + } + _selected_value_sizes.push_back(value.size); + } + decoded += batch_size; + } + cursor = range.first + range.count; + } + DORIS_CHECK(selection.total_values >= cursor); + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + + _string_refs.resize(selection.selected_values); + size_t byte_offset = 0; + for (size_t output = 0; output < _selected_value_sizes.size(); ++output) { + const size_t value_size = _selected_value_sizes[output]; + const char* value_data = + value_size == 0 ? nullptr : _selected_binary_values.data() + byte_offset; + _string_refs[output] = StringRef(value_data, value_size); + byte_offset += value_size; + } + DORIS_CHECK_EQ(_selected_value_sizes.size(), selection.selected_values); + DORIS_CHECK_EQ(byte_offset, _selected_binary_values.size()); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + RETURN_IF_ERROR(_validate_fixed_width_values()); + const size_t byte_size = num_values * static_cast(_type_length); + _fixed_values.resize(byte_size); + for (size_t row = 0; row < num_values; ++row) { + memcpy(_fixed_values.data() + row * _type_length, _values[row].data, _type_length); + } + return consumer.consume(_fixed_values.data(), num_values, + static_cast(_type_length)); + } + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.selected_values > + std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet delta-byte-array selection byte size overflows"); + } + _fixed_values.resize(selection.selected_values * value_width); + size_t output = 0; + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + // Skips validate widths too, so filtering cannot hide a malformed value after the + // decoder cursor has advanced past it. + RETURN_IF_ERROR(skip_values(range.first - cursor)); + size_t decoded = 0; + while (decoded < range.count) { + const size_t batch_size = _bounded_reconstruction_batch_size(range.count - decoded); + RETURN_IF_ERROR(_decode_slices(batch_size)); + RETURN_IF_ERROR(_validate_fixed_width_values()); + for (const auto& value : _values) { + memcpy(_fixed_values.data() + output * value_width, value.data, value_width); + ++output; + } + decoded += batch_size; + } + cursor = range.first + range.count; + } + DORIS_CHECK(selection.total_values >= cursor); + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_fixed_values.data(), output, value_width); + } + + Status set_data(Slice* slice) override { + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + auto prefix_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_prefix_len_decoder.set_bit_reader(prefix_reader)); + + // get the number of encoded prefix lengths + int num_prefix = _prefix_len_decoder.valid_values_count(); + DeltaBitPackDecoder prefix_locator; + prefix_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(prefix_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector ignored_prefixes( + std::min(static_cast(num_prefix), kLocateBatchSize)); + size_t located = 0; + while (located < static_cast(num_prefix)) { + const size_t batch_size = + std::min(static_cast(num_prefix) - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR(prefix_locator.decode(ignored_prefixes.data(), + static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + located += batch_size; + } + _num_valid_values = num_prefix; + + // at this time, the decoder_ will be at the start of the encoded suffix data. + RETURN_IF_ERROR(_suffix_decoder.set_bit_reader(_bit_reader)); + if (UNLIKELY(_suffix_decoder.valid_values_count() != num_prefix)) { + return Status::Corruption("Parquet delta prefix and suffix counts differ"); + } + + // TODO: read corrupted files written with bug(PARQUET-246). _last_value should be set + // to _last_value_in_previous_page when decoding a new page(except the first page) + _last_value = ""; + return Status::OK(); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_fixed_values, max_retained_bytes); + release_vector_if_oversized(&_selected_binary_values, max_retained_bytes); + release_vector_if_oversized(&_selected_value_sizes, max_retained_bytes); + release_vector_if_oversized(&_buffered_prefix_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _prefix_len_decoder.release_scratch(max_retained_bytes); + _suffix_decoder.release_scratch(max_retained_bytes); + // Prefix reconstruction crosses batch boundaries, so the previous value remains semantic + // decoder state until every value in the current page has been consumed. + if (_num_valid_values == 0) { + if (_last_value.capacity() > max_retained_bytes) std::string().swap(_last_value); + if (_last_value_in_previous_page.capacity() > max_retained_bytes) { + std::string().swap(_last_value_in_previous_page); + } + } + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _fixed_values.capacity() * sizeof(uint8_t) + + _selected_binary_values.capacity() * sizeof(char) + + _selected_value_sizes.capacity() * sizeof(size_t) + + _buffered_prefix_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _last_value.capacity() + + _last_value_in_previous_page.capacity() + + _prefix_len_decoder.retained_scratch_bytes() + + _suffix_decoder.retained_scratch_bytes(); + } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _fixed_values.size() * sizeof(uint8_t) + + _selected_binary_values.size() * sizeof(char) + + _selected_value_sizes.size() * sizeof(size_t) + + _buffered_prefix_length.size() * sizeof(int32_t) + + _buffered_data.size() * sizeof(char) + _last_value.size() + + _last_value_in_previous_page.size() + _prefix_len_decoder.active_scratch_bytes() + + _suffix_decoder.active_scratch_bytes(); + } + +private: + size_t _bounded_reconstruction_batch_size(size_t remaining_values) const { + constexpr size_t kReconstructionByteBudget = 4UL << 20; + constexpr size_t kMaxReconstructionValues = 4096; + // Prefix expansion, rather than encoded page bytes, determines reconstruction memory. A + // count-based chunk could turn one long value into gigabytes of temporary copies, so first + // learn its width and then cap subsequent chunks by a byte budget. + if (_last_value.empty()) { + return 1; + } + const size_t budgeted_values = + std::max(1, kReconstructionByteBudget / _last_value.size()); + return std::min({remaining_values, kMaxReconstructionValues, budgeted_values}); + } + + Status _skip_slices(size_t num_values) { + _values.resize(num_values); + int suffixes_decoded = 0; + RETURN_IF_ERROR(_suffix_decoder.decode(_values.data(), cast_set(num_values), + &suffixes_decoded)); + if (UNLIKELY(suffixes_decoded != num_values)) { + return Status::IOError("Expected to skip {} Parquet delta suffixes, skipped {}", + num_values, suffixes_decoded); + } + _buffered_prefix_length.resize(num_values); + uint32_t prefixes_decoded = 0; + RETURN_IF_ERROR(_prefix_len_decoder.decode( + _buffered_prefix_length.data(), cast_set(num_values), &prefixes_decoded)); + if (UNLIKELY(prefixes_decoded != num_values)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + + // Only the preceding value is semantic decoder state. Rebuilding every skipped value into + // one aggregate buffer would amplify compact full-prefix references into page_size * rows. + for (size_t row = 0; row < num_values; ++row) { + const int32_t encoded_prefix_size = _buffered_prefix_length[row]; + if (UNLIKELY(encoded_prefix_size < 0)) { + return Status::InvalidArgument("negative prefix length in DELTA_BYTE_ARRAY"); + } + const size_t prefix_size = static_cast(encoded_prefix_size); + if (UNLIKELY(prefix_size > _last_value.size())) { + return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); + } + size_t reconstructed_size = 0; + if (UNLIKELY( + common::add_overflow(prefix_size, _values[row].size, reconstructed_size))) { + return Status::InvalidArgument("excess expansion in DELTA_BYTE_ARRAY"); + } + if (_type_length > 0 && + UNLIKELY(reconstructed_size != static_cast(_type_length))) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + reconstructed_size, _type_length); + } + // resize() preserves the prefix already stored in _last_value, so only the suffix + // needs copying and repeated full-prefix values allocate no additional payload buffer. + _last_value.resize(reconstructed_size); + if (_values[row].size > 0) { + memcpy(_last_value.data() + prefix_size, _values[row].data, _values[row].size); + } + } + _num_valid_values -= cast_set(num_values); + if (_num_valid_values == 0) { + _last_value_in_previous_page = _last_value; + } + return Status::OK(); + } + + Status _validate_fixed_width_values() const { + if (_type_length <= 0) { + return Status::OK(); + } + const size_t value_width = static_cast(_type_length); + for (const auto& value : _values) { + if (UNLIKELY(value.size != value_width)) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + value.size, value_width); + } + } + return Status::OK(); + } + + Status _decode_slices(size_t num_values) { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-byte-array values, decoded {}", + num_values, decoded_count); + } + return Status::OK(); + } + + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::vector _fixed_values; + std::vector _selected_binary_values; + std::vector _selected_value_sizes; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _prefix_len_decoder; + DeltaLengthByteArrayDecoder _suffix_decoder; + std::string _last_value; + // string buffer for last value in previous page + std::string _last_value_in_previous_page; + int _num_valid_values; + std::vector _buffered_prefix_length; + std::vector _buffered_data; +}; +} // namespace doris::format::parquet::native + +namespace doris::format::parquet::native { + +template +Status DeltaBitPackDecoder::_init_header() { + if (!_bit_reader->GetVlqInt(&_values_per_block) || + !_bit_reader->GetVlqInt(&_mini_blocks_per_block) || + !_bit_reader->GetVlqInt(&_total_value_count) || + !_bit_reader->GetZigZagVlqInt(&_last_value)) { + return Status::IOError("Init header eof"); + } + if (_values_per_block == 0) { + return Status::InvalidArgument("Cannot have zero value per block"); + } + if (_values_per_block % 128 != 0) { + return Status::InvalidArgument( + "the number of values in a block must be multiple of 128, but it's " + + std::to_string(_values_per_block)); + } + if (_mini_blocks_per_block == 0) { + return Status::InvalidArgument("Cannot have zero miniblock per block"); + } + // Parquet requires integral block geometry; truncating here would silently decode fewer + // values than the header declares and desynchronize the following block. + if (UNLIKELY(_values_per_block % _mini_blocks_per_block != 0)) { + return Status::Corruption("Parquet delta block size {} is not divisible by {} miniblocks", + _values_per_block, _mini_blocks_per_block); + } + _values_per_mini_block = _values_per_block / _mini_blocks_per_block; + if (_values_per_mini_block == 0) { + return Status::InvalidArgument("Cannot have zero value per miniblock"); + } + if (_values_per_mini_block % 32 != 0) { + return Status::InvalidArgument( + "The number of values in a miniblock must be multiple of 32, but it's " + + std::to_string(_values_per_mini_block)); + } + // Encoded counts are external ULEB32 values. Bound them by the page's advertised logical + // values before any vector uses the count; optional levels may only reduce this upper bound. + if (UNLIKELY(_total_value_count > _expected_values)) { + return Status::Corruption("Parquet delta header advertises {} values, page allows {}", + _total_value_count, _expected_values); + } + _total_values_remaining = _total_value_count; + _delta_bit_widths.clear(); + // init as empty property + _block_initialized = false; + _delta_bit_width = 0; + _values_remaining_current_mini_block = 0; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_block() { + DCHECK_GT(_total_values_remaining, 0) << "InitBlock called at EOF"; + if (!_bit_reader->GetZigZagVlqInt(&_min_delta)) { + return Status::IOError("Init block eof"); + } + + // One byte follows for each miniblock. Defer allocation until a block is actually consumed so + // a one-value page cannot turn an irrelevant malicious miniblock count into a large resize. + if (UNLIKELY(_mini_blocks_per_block > static_cast(_bit_reader->bytes_left()))) { + return Status::Corruption("Parquet delta miniblock count {} exceeds remaining {} bytes", + _mini_blocks_per_block, _bit_reader->bytes_left()); + } + _delta_bit_widths.resize(_mini_blocks_per_block); + + // read the bitwidth of each miniblock + uint8_t* bit_width_data = _delta_bit_widths.data(); + for (uint32_t i = 0; i < _mini_blocks_per_block; ++i) { + if (!_bit_reader->GetAligned(1, bit_width_data + i)) { + return Status::IOError("Decode bit-width EOF"); + } + // Note that non-conformant bitwidth entries are allowed by the Parquet spec + // for extraneous miniblocks in the last block (GH-14923), so we check + // the bitwidths when actually using them (see InitMiniBlock()). + } + _mini_block_idx = 0; + _block_initialized = true; + RETURN_IF_ERROR(_init_mini_block(bit_width_data[0])); + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_mini_block(int bit_width) { + if (bit_width > kMaxDeltaBitWidth) [[unlikely]] { + return Status::InvalidArgument("delta bit width larger than integer bit width"); + } + _delta_bit_width = bit_width; + _values_remaining_current_mini_block = _values_per_mini_block; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_get_internal(T* buffer, uint32_t num_values, + uint32_t* out_num_values) { + num_values = std::min(num_values, _total_values_remaining); + if (num_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + uint32_t i = 0; + while (i < num_values) { + if (_values_remaining_current_mini_block == 0) [[unlikely]] { + if (!_block_initialized) [[unlikely]] { + buffer[i++] = _last_value; + DCHECK_EQ(i, 1); // we're at the beginning of the page + if (i == num_values) { + // When block is uninitialized and i reaches num_values we have two + // different possibilities: + // 1. _total_value_count == 1, which means that the page may have only + // one value (encoded in the header), and we should not initialize + // any block. + // 2. _total_value_count != 1, which means we should initialize the + // incoming block for subsequent reads. + if (_total_value_count != 1) { + RETURN_IF_ERROR(_init_block()); + } + break; + } + RETURN_IF_ERROR(_init_block()); + } else { + ++_mini_block_idx; + if (_mini_block_idx < _mini_blocks_per_block) { + RETURN_IF_ERROR(_init_mini_block(_delta_bit_widths.data()[_mini_block_idx])); + } else { + RETURN_IF_ERROR(_init_block()); + } + } + } + + uint32_t values_decode = std::min(_values_remaining_current_mini_block, num_values - i); + for (uint32_t j = 0; j < values_decode; ++j) { + if (!_bit_reader->GetValue(_delta_bit_width, buffer + i + j)) { + return Status::IOError("Get batch EOF"); + } + } + for (int j = 0; j < values_decode; ++j) { + // Addition between min_delta, packed int and last_value should be treated as + // unsigned addition. Overflow is as expected. + buffer[i + j] = static_cast(_min_delta) + static_cast(buffer[i + j]) + + static_cast(_last_value); + _last_value = buffer[i + j]; + } + _values_remaining_current_mini_block -= values_decode; + i += values_decode; + } + _total_values_remaining -= num_values; + + if (_total_values_remaining == 0) [[unlikely]] { + int64_t padding_bits = 0; + // Footer-controlled miniblock counts can exceed 32-bit products. Widen before multiplying; + // BitReader::Advance then verifies that the full declared padding exists in the stream. + RETURN_IF_ERROR(detail::checked_delta_padding_bits(cast_set(_delta_bit_width), + _values_remaining_current_mini_block, + &padding_bits)); + if (!_bit_reader->Advance(padding_bits)) { + return Status::IOError("Skip padding EOF"); + } + _values_remaining_current_mini_block = 0; + } + *out_num_values = num_values; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp new file mode 100644 index 00000000000000..1f1166421a4035 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { + +// Dictionary decoders retain only encoded physical values and the index-stream cursor. Logical +// interpretation is deliberately delegated to DataTypeSerDe through decode_dictionary(). +template +class FixLengthDictDecoder final : public BaseDictDecoder { +public: + FixLengthDictDecoder() = default; + ~FixLengthDictDecoder() override = default; + + size_t dictionary_size() const override { return _num_dictionary_values; } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return fixed_consumer.consume(_dict.get(), _num_dictionary_values, + static_cast(_type_length)); + } + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override { + if (UNLIKELY(_type_length <= 0 || length < 0 || + num_values > std::numeric_limits::max() / + static_cast(_type_length) || + num_values * static_cast(_type_length) != + static_cast(length))) { + return Status::Corruption("Wrong dictionary data for fixed length type"); + } + if (UNLIKELY(dict == nullptr)) { + return Status::Corruption("Fixed-length Parquet dictionary is null"); + } + _dict = std::move(dict); + _num_dictionary_values = num_values; + ++_dictionary_generation; + return Status::OK(); + } + +private: + size_t _num_dictionary_values = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp new file mode 100644 index 00000000000000..28d8154f9db10d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +#include + +#include "util/cpu_info.h" + +namespace doris::format::parquet::native { + +Status FixLengthPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + // Truncated PLAIN pages retain the public error keyword used by corrupt-file regression + // checks, while the bounds check still prevents the native decoder from reading past data. + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); + } + RETURN_IF_ERROR(consumer.consume(reinterpret_cast(_data->data) + _offset, + num_values, static_cast(_type_length))); + _offset += byte_size; + return Status::OK(); +} + +Status FixLengthPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet plain selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError("Unexpected end of stream in Parquet plain selection decoder"); + } + const auto* values = reinterpret_cast(_data->data) + _offset; + _offset += input_bytes; + const size_t selected_bytes = selection.selected_values * value_width; + constexpr size_t MIN_FRAGMENTED_RANGES = 8; + constexpr size_t MAX_AVERAGE_RANGE_VALUES = 4; + const bool fragmented = + selection.ranges.size() >= MIN_FRAGMENTED_RANGES && + selection.selected_values <= selection.ranges.size() * MAX_AVERAGE_RANGE_VALUES; + if (fragmented && selected_bytes <= static_cast(CpuInfo::get_l2_cache_size())) { + _selected_values.resize(selected_bytes); + size_t output_offset = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first + range.count <= selection.total_values); + const size_t range_bytes = range.count * value_width; + memcpy(_selected_values.data() + output_offset, values + range.first * value_width, + range_bytes); + output_offset += range_bytes; + } + DORIS_CHECK_EQ(output_offset, selected_bytes); + // Tiny alternating runs make every consumer re-enter conversion and grow its destination + // column per range. A cache-resident gather keeps the page access sequential and restores + // one atomic conversion batch without retaining page-sized scratch for dense selections. + return consumer.consume(_selected_values.data(), selection.selected_values, value_width); + } + _selected_values.clear(); + // PLAIN pages are random-access fixed-width spans. Keep those page bytes pinned while the + // consumer gathers directly into the final column, otherwise sparse scans pay for a second + // selected-width buffer and copy before materialization. + return consumer.consume_selected(values, value_width, selection.ranges); +} + +Status FixLengthPlainDecoder::skip_values(size_t num_values) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || num_values > (_data->size - _offset) / value_width)) { + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); + } + _offset += num_values * value_width; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h new file mode 100644 index 00000000000000..14913217e2e0de --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { + +class FixLengthPlainDecoder final : public Decoder { +public: + FixLengthPlainDecoder() = default; + ~FixLengthPlainDecoder() override = default; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_selected_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { return _selected_values.capacity(); } + size_t active_scratch_bytes() const override { return _selected_values.size(); } + +private: + std::vector _selected_values; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.cpp b/be/src/format_v2/parquet/reader/native/level_decoder.cpp new file mode 100644 index 00000000000000..618d6ebf9f6d45 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.cpp @@ -0,0 +1,271 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_decoder.h" + +#include + +#include +#include + +#include "common/cast_set.h" +#include "util/bit_stream_utils.inline.h" +#include "util/bit_util.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { + +static constexpr size_t V1_LEVEL_SIZE = 4; + +Status LevelDecoder::init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels) { + _encoding = encoding; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; + switch (encoding) { + case tparquet::Encoding::RLE: { + if (slice->size < V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + + uint8_t* data = (uint8_t*)slice->data; + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > slice->size - V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + _rle_decoder = RleBatchDecoder(data + V1_LEVEL_SIZE, num_bytes, _bit_width); + + slice->data += V1_LEVEL_SIZE + num_bytes; + slice->size -= V1_LEVEL_SIZE + num_bytes; + break; + } + case tparquet::Encoding::BIT_PACKED: { + // Header counts are uint32_t and can overflow before the byte bound check. Widen first so + // a forged level count cannot wrap to a small slice and desynchronize following values. + const uint64_t num_bits = static_cast(num_levels) * _bit_width; + const size_t num_bytes = static_cast((num_bits + 7) / 8); + if (num_bytes > slice->size) { + return Status::Corruption("Wrong parquet level format"); + } + if (num_bytes > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet BIT_PACKED level stream is too large"); + } + _bit_packed_decoder = BitReader((uint8_t*)slice->data, cast_set(num_bytes)); + + slice->data += num_bytes; + slice->size -= num_bytes; + break; + } + default: + return Status::IOError("Unsupported encoding for parquet level"); + } + return Status::OK(); +} + +Status LevelDecoder::init_v2(const Slice& levels, level_t max_level, uint32_t num_levels) { + _encoding = tparquet::Encoding::RLE; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; + size_t byte_length = levels.size; + _rle_decoder = RleBatchDecoder((uint8_t*)levels.data, cast_set(byte_length), + _bit_width); + return Status::OK(); +} + +size_t LevelDecoder::get_levels(level_t* levels, size_t n) { + _can_rewind = false; + // toto template. + if (_encoding == tparquet::Encoding::RLE) { + n = std::min((size_t)_num_levels, n); + size_t num_decoded = 0; + if (_has_buffered_level && n > 0) { + if (!accept_level(_buffered_level)) { + return 0; + } + levels[num_decoded++] = _buffered_level; + _has_buffered_level = false; + } + if (num_decoded < n) { + const size_t remaining = n - num_decoded; + _rle_scratch.resize(remaining); + const size_t batch_decoded = + _rle_decoder.GetBatch(_rle_scratch.data(), cast_set(remaining)); + for (size_t i = 0; i < batch_decoded; ++i) { + const level_t level = cast_set(_rle_scratch[i]); + if (!accept_level(level)) { + return 0; + } + levels[num_decoded + i] = level; + } + num_decoded += batch_decoded; + } + _num_levels -= num_decoded; + _can_rewind = false; + return num_decoded; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + n = std::min((size_t)_num_levels, n); + size_t decoded = 0; + for (; decoded < n; ++decoded) { + level_t level = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &level)) { + break; + } + if (!accept_level(level)) { + return 0; + } + levels[decoded] = level; + } + _num_levels -= decoded; + return decoded; + } + return 0; +} + +size_t LevelDecoder::get_next_run(level_t* val, size_t max_run) { + DORIS_CHECK(val != nullptr); + _can_rewind = false; + max_run = std::min(max_run, _num_levels); + if (max_run == 0) { + return 0; + } + if (_encoding == tparquet::Encoding::RLE) { + size_t decoded = 0; + if (_has_buffered_level) { + *val = _buffered_level; + _has_buffered_level = false; + if (!accept_level(*val)) { + return 0; + } + decoded = 1; + } else { + uint16_t first = 0; + if (_rle_decoder.GetBatch(&first, 1) != 1) { + return 0; + } + *val = cast_set(first); + if (!accept_level(*val)) { + return 0; + } + decoded = 1; + } + while (decoded < max_run) { + const int32_t repeats = _rle_decoder.NextNumRepeats(); + if (repeats > 0) { + const level_t repeated = cast_set(_rle_decoder.GetRepeatedValue(0)); + if (!accept_level(repeated)) return 0; + if (repeated != *val) break; + const int32_t consume = std::min(repeats, cast_set(max_run - decoded)); + _rle_decoder.GetRepeatedValue(consume); + decoded += consume; + continue; + } + if (_rle_decoder.NextNumLiterals() == 0) break; + uint16_t literal = 0; + if (!_rle_decoder.GetLiteralValues(1, &literal)) break; + const level_t next = cast_set(literal); + if (!accept_level(next)) return 0; + if (next != *val) { + // Batch RLE has no physical rewind; retain the one-value lookahead logically. + _buffered_level = next; + _has_buffered_level = true; + break; + } + ++decoded; + } + _num_levels -= decoded; + _can_rewind = false; + return decoded; + } + if (_encoding != tparquet::Encoding::BIT_PACKED || + !_bit_packed_decoder.GetValue(_bit_width, val)) { + return 0; + } + if (!accept_level(*val)) { + return 0; + } + size_t decoded = 1; + while (decoded < max_run) { + level_t next = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &next)) { + break; + } + if (!accept_level(next)) return 0; + if (next != *val) { + // The lookahead belongs to the following run, so cursor APIs must leave it unread. + _bit_packed_decoder.Rewind(_bit_width); + break; + } + ++decoded; + } + _num_levels -= decoded; + return decoded; +} + +level_t LevelDecoder::get_next() { + if (_num_levels == 0) { + return -1; + } + level_t next = -1; + bool decoded = false; + if (_encoding == tparquet::Encoding::RLE) { + if (_has_buffered_level) { + next = _buffered_level; + _has_buffered_level = false; + decoded = true; + } else { + uint16_t value = 0; + decoded = _rle_decoder.GetBatch(&value, 1) == 1; + next = cast_set(value); + } + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + decoded = _bit_packed_decoder.GetValue(_bit_width, &next); + } + if (!decoded) { + return -1; + } + if (!accept_level(next)) { + return -1; + } + --_num_levels; + _last_level = next; + _can_rewind = true; + return next; +} + +void LevelDecoder::rewind_one() { + if (_encoding == tparquet::Encoding::RLE) { + DORIS_CHECK(_can_rewind && !_has_buffered_level); + _buffered_level = _last_level; + _has_buffered_level = true; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + DORIS_CHECK(_can_rewind); + _bit_packed_decoder.Rewind(_bit_width); + } else { + return; + } + // Rewinding restores one advertised level as well as its encoded bits. + ++_num_levels; + _can_rewind = false; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.h b/be/src/format_v2/parquet/reader/native/level_decoder.h new file mode 100644 index 00000000000000..9dcff491eb0759 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.h @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/status.h" +#include "format_v2/parquet/reader/native/common.h" +#include "util/bit_stream_utils.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +class LevelDecoder { +public: + LevelDecoder() = default; + ~LevelDecoder() = default; + + Status init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels); + + Status init_v2(const Slice& levels, level_t max_level, uint32_t num_levels); + + inline bool has_levels() const { return _num_levels > 0; } + + size_t get_levels(level_t* levels, size_t n); + + size_t get_next_run(level_t* val, size_t max_run); + + level_t get_next(); + + void rewind_one(); + + void release_scratch(size_t max_retained_bytes) { + if (_rle_scratch.capacity() * sizeof(uint16_t) > max_retained_bytes) { + std::vector().swap(_rle_scratch); + } + } + size_t retained_scratch_bytes() const { return _rle_scratch.capacity() * sizeof(uint16_t); } + size_t active_scratch_bytes() const { return _rle_scratch.size() * sizeof(uint16_t); } + +private: + bool accept_level(level_t level) { + if (level >= 0 && level <= _max_level) { + return true; + } + // Bit width rounds up to a power of two, so encoded values can fit the bit stream while + // still exceeding the schema maximum. Poison the decoder before any caller can use them. + _num_levels = 0; + _has_buffered_level = false; + _can_rewind = false; + return false; + } + + tparquet::Encoding::type _encoding; + level_t _bit_width = 0; + level_t _max_level = 0; + uint32_t _num_levels = 0; + RleBatchDecoder _rle_decoder; + std::vector _rle_scratch; + BitReader _bit_packed_decoder; + bool _has_buffered_level = false; + bool _can_rewind = false; + level_t _buffered_level = -1; + level_t _last_level = -1; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.cpp b/be/src/format_v2/parquet/reader/native/level_reader.cpp new file mode 100644 index 00000000000000..06b71582d7a2be --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.cpp @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_reader.h" + +#include +#include + +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/tracing_file_reader.h" + +namespace doris::format::parquet::native { + +class LevelReader::Impl { +public: + virtual ~Impl() = default; + + virtual Status init() = 0; + virtual Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) = 0; + virtual ColumnChunkReaderStatistics statistics() = 0; +}; + +template +class LevelReaderImpl final : public LevelReader::Impl { +public: + LevelReaderImpl(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, std::string page_cache_file_key, + ParquetReaderCompat compat) + : _file(std::move(file)), + _column_chunk(std::move(column_chunk)), + _field(field), + _total_rows(total_rows), + _max_buffer_size(max_buffer_size), + _io_ctx(io_ctx), + _enable_page_cache(enable_page_cache), + _page_cache_file_key(std::move(page_cache_file_key)), + _compat(compat) {} + + Status init() override { + DORIS_CHECK(_file != nullptr); + DORIS_CHECK(_field != nullptr); + const auto& metadata = _column_chunk.meta_data; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(metadata, _file->size(), + _compat.parquet_816_padding, &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_size = chunk_range.length; + size_t prefetch_buffer_size = std::min(chunk_size, _max_buffer_size); + auto* tracing_reader = typeid_cast(_file.get()); + if ((tracing_reader != nullptr && + typeid_cast(tracing_reader->inner_reader().get()) != + nullptr) || + typeid_cast(_file.get()) != nullptr) { + prefetch_buffer_size = 0; + } + _stream = std::make_unique(_file, chunk_start, chunk_size, + prefetch_buffer_size); + _chunk_reader = std::make_unique>( + _stream.get(), &_column_chunk, _field, nullptr, _total_rows, _io_ctx, + ParquetPageReadContext(_enable_page_cache, _page_cache_file_key, + _compat.data_page_v2_always_compressed), + &chunk_range); + return _chunk_reader->init(); + } + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) override { + DORIS_CHECK(repetition_levels != nullptr); + DORIS_CHECK(definition_levels != nullptr); + DORIS_CHECK(rows_read != nullptr); + repetition_levels->clear(); + definition_levels->clear(); + *rows_read = 0; + if (_current_row > _total_rows || rows > _total_rows - _current_row) { + return Status::Corruption("Parquet level reader requested rows [{}, {}) of {}", + _current_row, _current_row + rows, _total_rows); + } + if constexpr (IN_COLLECTION) { + RETURN_IF_ERROR( + read_nested_rows(rows, repetition_levels, definition_levels, rows_read)); + } else { + RETURN_IF_ERROR(read_flat_rows(rows, repetition_levels, definition_levels, rows_read)); + } + _current_row += *rows_read; + return Status::OK(); + } + + ColumnChunkReaderStatistics statistics() override { return _chunk_reader->statistics(); } + +private: + Status read_flat_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + const size_t read_now = std::min( + rows - *rows_read, static_cast(_chunk_reader->remaining_num_values())); + if (read_now == 0) { + // Zero-value data pages are legal and do not contribute to the chunk cardinality. + // Advance them before applying the no-progress guard used for truncated chunks. + if (_chunk_reader->remaining_num_values() == 0 && _chunk_reader->has_next_page()) { + RETURN_IF_ERROR(_chunk_reader->next_page()); + continue; + } + return Status::Corruption("Parquet flat level reader made no progress"); + } + RETURN_IF_ERROR( + _chunk_reader->read_levels(read_now, repetition_levels, definition_levels)); + *rows_read += read_now; + if (_chunk_reader->remaining_num_values() == 0 && _chunk_reader->has_next_page()) { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + return Status::OK(); + } + + Status read_nested_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(_current_row + *rows_read)); + const size_t start_level = definition_levels->size(); + size_t loaded_rows = 0; + bool crosses_page = false; + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows( + *repetition_levels, rows - *rows_read, &loaded_rows, &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(*definition_levels, start_level)); + while (crosses_page) { + const size_t continuation_start = definition_levels->size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(*repetition_levels, + &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR( + _chunk_reader->skip_nested_values(*definition_levels, continuation_start)); + } + if (loaded_rows == 0) { + return Status::Corruption("Parquet nested level reader made no progress"); + } + *rows_read += loaded_rows; + } + return Status::OK(); + } + + io::FileReaderSPtr _file; + tparquet::ColumnChunk _column_chunk; + NativeFieldSchema* _field = nullptr; + size_t _total_rows = 0; + size_t _max_buffer_size = 0; + io::IOContext* _io_ctx = nullptr; + bool _enable_page_cache = false; + std::string _page_cache_file_key; + ParquetReaderCompat _compat; + size_t _current_row = 0; + std::unique_ptr _stream; + std::unique_ptr> _chunk_reader; +}; + +Status LevelReader::create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, + std::unique_ptr* reader) { + DORIS_CHECK(reader != nullptr); + DORIS_CHECK(field != nullptr); + std::unique_ptr impl; + if (field->repetition_level > 0) { + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); + } else { + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); + } + RETURN_IF_ERROR(impl->init()); + reader->reset(new LevelReader(std::move(impl))); + return Status::OK(); +} + +LevelReader::LevelReader(std::unique_ptr impl) : _impl(std::move(impl)) {} + +LevelReader::~LevelReader() = default; + +Status LevelReader::read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + return _impl->read_rows(rows, repetition_levels, definition_levels, rows_read); +} + +Status LevelReader::skip_rows(size_t rows) { + size_t rows_read = 0; + RETURN_IF_ERROR( + read_rows(rows, &_skip_repetition_levels, &_skip_definition_levels, &rows_read)); + if (rows_read != rows) { + return Status::Corruption("Parquet level reader skipped {} of {} rows", rows_read, rows); + } + return Status::OK(); +} + +ColumnChunkReaderStatistics LevelReader::statistics() { + return _impl->statistics(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.h b/be/src/format_v2/parquet/reader/native/level_reader.h new file mode 100644 index 00000000000000..11117d25dcb252 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.h @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris::io { +struct IOContext; +} // namespace doris::io + +namespace doris::format::parquet::native { + +// A physical-leaf reader that advances all three Parquet streams (repetition levels, definition +// levels and encoded values) but retains only the two level streams. It is used for shape-only +// operations such as COUNT(nullable_col), where materializing a large BYTE_ARRAY value would be +// both unnecessary and potentially unbounded. +// +// This deliberately shares ColumnChunkReader with the value path. Page V1/V2 parsing, +// decompression, dictionary-page handling, page cache semantics and corruption checks therefore +// cannot drift between an aggregate shortcut and an ordinary scan. +class LevelReader { +public: + class Impl; + + static Status create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, + std::unique_ptr* reader); + + ~LevelReader(); + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read); + Status skip_rows(size_t rows); + ColumnChunkReaderStatistics statistics(); + +private: + explicit LevelReader(std::unique_ptr impl); + + std::unique_ptr _impl; + // COUNT range gaps can call skip repeatedly as the adaptive row cap changes. Keep these + // throw-away level buffers on the persistent reader so only their logical sizes are reset. + std::vector _skip_repetition_levels; + std::vector _skip_definition_levels; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp new file mode 100644 index 00000000000000..37f682cbd4d9d5 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -0,0 +1,332 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/page_reader.h" + +#include +#include +#include +#include + +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/slice.h" +#include "util/thrift_util.h" + +namespace doris { +namespace io { +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { +static constexpr size_t INIT_PAGE_HEADER_SIZE = 128; + +template +Status PageReader::_validate_page_header(uint32_t header_size) const { + if (UNLIKELY(_cur_page_header.compressed_page_size < 0 || + _cur_page_header.uncompressed_page_size < 0)) { + return Status::Corruption("Parquet page has a negative compressed or uncompressed size"); + } + if (UNLIKELY(header_size > _end_offset - _offset || + static_cast(_cur_page_header.compressed_page_size) > + _end_offset - _offset - header_size)) { + // Sizes are untrusted signed Thrift fields and must fit the column chunk before arithmetic. + return Status::Corruption("Parquet page payload exceeds its column chunk"); + } + + const bool has_v1 = _cur_page_header.__isset.data_page_header; + const bool has_v2 = _cur_page_header.__isset.data_page_header_v2; + const bool has_dictionary = _cur_page_header.__isset.dictionary_page_header; + const bool has_index = _cur_page_header.__isset.index_page_header; + bool matching_layout = false; + switch (_cur_page_header.type) { + case tparquet::PageType::DATA_PAGE: + matching_layout = has_v1 && !has_v2 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DATA_PAGE_V2: + matching_layout = has_v2 && !has_v1 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DICTIONARY_PAGE: + matching_layout = has_dictionary && !has_v1 && !has_v2 && !has_index; + break; + case tparquet::PageType::INDEX_PAGE: + matching_layout = has_index && !has_v1 && !has_v2 && !has_dictionary; + break; + default: + // Forward-compatible auxiliary pages have no known page-specific member. Their bounded + // payload can be skipped safely; accepting a known member here would let the enum and + // decoder layout disagree. + matching_layout = !has_v1 && !has_v2 && !has_dictionary && !has_index; + break; + } + if (UNLIKELY(!matching_layout)) { + // The type discriminant owns the only valid union member. Guessing from an optional header + // lets a malformed page use one layout for validation and another for decoding. + return Status::Corruption("Parquet page type does not match its page-header layout"); + } + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + const auto& v2 = _cur_page_header.data_page_header_v2; + if (UNLIKELY(v2.num_values < 0 || v2.num_rows < 0 || v2.num_nulls < 0 || + v2.repetition_levels_byte_length < 0 || + v2.definition_levels_byte_length < 0)) { + return Status::Corruption("Parquet data page v2 has negative counts or level sizes"); + } + if (UNLIKELY(v2.num_nulls > v2.num_values || v2.num_rows > v2.num_values)) { + return Status::Corruption( + "Parquet data page v2 null or row count exceeds its value count"); + } + const uint64_t level_bytes = static_cast(v2.repetition_levels_byte_length) + + static_cast(v2.definition_levels_byte_length); + if (UNLIKELY(level_bytes > static_cast(_cur_page_header.compressed_page_size) || + level_bytes > + static_cast(_cur_page_header.uncompressed_page_size))) { + return Status::Corruption("Parquet data page v2 level bytes exceed the page payload"); + } + } else if (_cur_page_header.type == tparquet::PageType::DATA_PAGE && + UNLIKELY(_cur_page_header.data_page_header.num_values < 0)) { + return Status::Corruption("Parquet data page has a negative value count"); + } else if (_cur_page_header.type == tparquet::PageType::DICTIONARY_PAGE && + UNLIKELY(_cur_page_header.dictionary_page_header.num_values < 0)) { + return Status::Corruption("Parquet dictionary page has a negative value count"); + } + return Status::OK(); +} + +template +PageReader::PageReader(io::BufferedStreamReader* reader, + io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, + const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index) + : _reader(reader), + _io_ctx(io_ctx), + _offset(offset), + _start_offset(offset), + _end_offset(offset + length), + _total_rows(total_rows), + _metadata(metadata), + _page_read_ctx(page_read_ctx), + _offset_index(offset_index) { + _next_header_offset = _offset; + _state = INITIALIZED; + _page_cache_key_builder.init(_page_read_ctx.page_cache_file_key); + + if constexpr (OFFSET_INDEX) { + _end_row = _offset_index != nullptr && _offset_index->page_locations.size() >= 2 + ? _offset_index->page_locations[1].first_row_index + : _total_rows; + } +} + +template +void PageReader::_reconcile_offset_index_location( + uint64_t header_offset, uint32_t header_size) { + if constexpr (!OFFSET_INDEX) { + return; + } + if (_offset_index == nullptr || _page_index >= _offset_index->page_locations.size()) { + return; + } + const auto& location = _offset_index->page_locations[_page_index]; + const bool data_page = _cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2; + const uint64_t serialized_size = static_cast(header_size) + + static_cast(_cur_page_header.compressed_page_size); + const bool matches = data_page && location.offset >= 0 && location.compressed_page_size > 0 && + header_offset == static_cast(location.offset) && + serialized_size == static_cast(location.compressed_page_size); + if (!matches && (data_page || (location.offset >= 0 && + header_offset >= static_cast(location.offset)))) { + // OffsetIndex is optional. Once its data-page rectangle disagrees with the serialized + // page, continue sequentially so a shifted location cannot redirect the next seek. + _offset_index = nullptr; + } +} + +template +void PageReader::_update_sequential_row_range() { + if constexpr (OFFSET_INDEX) { + if (_offset_index != nullptr) { + return; + } + } + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE) { + _end_row = _start_row + _cur_page_header.data_page_header.num_values; + } + } +} + +template +Status PageReader::parse_page_header() { + if (_state == HEADER_PARSED) { + return Status::OK(); + } + if (UNLIKELY(_offset < _start_offset || _offset >= _end_offset)) { + return Status::IOError("Out-of-bounds Access"); + } + if (UNLIKELY(_offset != _next_header_offset)) { + return Status::IOError("Wrong header position, should seek to a page header first"); + } + if (UNLIKELY(_state != INITIALIZED)) { + return Status::IOError("Should skip or load current page to get next page"); + } + + _page_statistics.page_read_counter += 1; + + // Parse page header from file; header bytes are saved for possible cache insertion + const uint8_t* page_header_buf = nullptr; + size_t max_size = _end_offset - _offset; + size_t header_size = std::min(INIT_PAGE_HEADER_SIZE, max_size); + const size_t MAX_PAGE_HEADER_SIZE = config::parquet_header_max_size_mb << 20; + uint32_t real_header_size = 0; + + // Try a header-only lookup in the page cache. Cached pages store + // header + optional v2 levels + uncompressed payload, so we can + // parse the page header directly from the cached bytes and avoid + // a file read for the header. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + PageCacheHandle handle; + StoragePageCache::CacheKey key = make_page_cache_key(static_cast(_offset)); + if (StoragePageCache::instance()->lookup(key, &handle, segment_v2::DATA_PAGE)) { + // Parse header directly from cached data + _page_cache_handle = std::move(handle); + Slice s = _page_cache_handle.data(); + real_header_size = cast_set(s.size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Thrift does not clear absent optional fields when reusing an output object. + _cur_page_header = tparquet::PageHeader {}; + auto st = deserialize_thrift_msg(reinterpret_cast(s.data), + &real_header_size, true, &_cur_page_header); + if (!st.ok()) return st; + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + _reconcile_offset_index_location(_offset, real_header_size); + _update_sequential_row_range(); + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } + // Increment page cache counters for a true cache hit on header+payload + _page_statistics.page_cache_hit_counter += 1; + // Detect whether the cached payload is compressed or decompressed and record + bool is_cache_payload_decompressed = should_cache_decompressed( + &_cur_page_header, _metadata, _page_read_ctx.data_page_v2_always_compressed); + + if (is_cache_payload_decompressed) { + _page_statistics.page_cache_decompressed_hit_counter += 1; + } else { + _page_statistics.page_cache_compressed_hit_counter += 1; + } + + _is_cache_payload_decompressed = is_cache_payload_decompressed; + + // Save header bytes for later use (e.g., to insert updated cache entries) + _header_buf.assign(s.data, s.data + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); + } else { + _page_statistics.page_cache_missing_counter += 1; + // Clear any existing cache handle on miss to avoid holding stale handle + _page_cache_handle = PageCacheHandle(); + } + } + // NOTE: page cache lookup for *decompressed* page data is handled in + // ColumnChunkReader::load_page_data(). PageReader should only be + // responsible for parsing the header bytes from the file and saving + // them in `_header_buf` for possible later insertion into the cache. + while (true) { + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + header_size = std::min(header_size, max_size); + { + SCOPED_RAW_TIMER(&_page_statistics.read_page_header_time); + RETURN_IF_ERROR(_reader->read_bytes(&page_header_buf, _offset, header_size, _io_ctx)); + } + real_header_size = cast_set(header_size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Reset on every retry as a partial deserialize can otherwise leak a stale union member. + _cur_page_header = tparquet::PageHeader {}; + auto st = + deserialize_thrift_msg(page_header_buf, &real_header_size, true, &_cur_page_header); + if (st.ok()) { + break; + } + if (_offset + header_size >= _end_offset || real_header_size > MAX_PAGE_HEADER_SIZE) { + return Status::IOError( + "Failed to deserialize parquet page header. offset: {}, " + "header size: {}, end offset: {}, real header size: {}", + _offset, header_size, _end_offset, real_header_size); + } + header_size <<= 2; + } + + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + _reconcile_offset_index_location(_offset, real_header_size); + _update_sequential_row_range(); + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } + + // Save header bytes for possible cache insertion later + _header_buf.assign(page_header_buf, page_header_buf + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status PageReader::get_page_data(Slice& slice) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::IOError("Should generate page header first to load current page data"); + } + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + slice.size = _cur_page_header.compressed_page_size; + RETURN_IF_ERROR(_reader->read_bytes(slice, _offset, _io_ctx)); + _offset += slice.size; + _state = DATA_LOADED; + return Status::OK(); +} + +template class PageReader; +template class PageReader; +template class PageReader; +template class PageReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.h b/be/src/format_v2/parquet/reader/native/page_reader.h new file mode 100644 index 00000000000000..da2c066891bda7 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.h @@ -0,0 +1,303 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/status.h" +#include "storage/cache/page_cache.h" +#include "util/block_compression.h" +namespace doris { +class BlockCompressionCodec; + +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +/** + * Use to deserialize parquet page header, and get the page data in iterator interface. + */ + +// Session-level options for parquet page reading/caching. +struct ParquetPageReadContext { + // A default-constructed context has no stable file identity, so cache lookup must stay off. + bool enable_parquet_file_page_cache = false; + std::string page_cache_file_key; + bool data_page_v2_always_compressed = false; + ParquetPageReadContext() = default; + ParquetPageReadContext(bool enable_parquet_file_page_cache, std::string page_cache_file_key, + bool data_page_v2_always_compressed = false) + : enable_parquet_file_page_cache(enable_parquet_file_page_cache && + !page_cache_file_key.empty()), + page_cache_file_key(std::move(page_cache_file_key)), + data_page_v2_always_compressed(data_page_v2_always_compressed) {} +}; + +inline bool should_cache_decompressed(const tparquet::PageHeader* header, + const tparquet::ColumnMetaData& metadata, + bool data_page_v2_always_compressed = false) { + // Data Page V2 declares its payload representation independently of the column codec. A warm + // hit must never send an explicitly uncompressed cached payload through the codec again. + if (header->__isset.data_page_header_v2 && !header->data_page_header_v2.is_compressed && + !data_page_v2_always_compressed) { + return true; + } + if (header->compressed_page_size <= 0) return true; + if (metadata.codec == tparquet::CompressionCodec::UNCOMPRESSED) return true; + if (header->uncompressed_page_size == 0) return true; + + double ratio = static_cast(header->uncompressed_page_size) / + static_cast(header->compressed_page_size); + return ratio <= config::parquet_page_cache_decompress_threshold; +} + +class ParquetPageCacheKeyBuilder { +public: + void init(std::string file_key) { _file_key_prefix = std::move(file_key); } + StoragePageCache::CacheKey make_key(uint64_t end_offset, int64_t offset) const { + return StoragePageCache::CacheKey(_file_key_prefix, end_offset, offset); + } + +private: + std::string _file_key_prefix; +}; + +template +class PageReader { +public: + struct PageStatistics { + int64_t decode_header_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; + }; + + PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index = nullptr); + ~PageReader() = default; + + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + if (_offset_index != nullptr) { + return _page_index + 1 < _offset_index->page_locations.size(); + } + } + return _offset < _end_offset; + } + + Status parse_page_header(); + + Status next_page() { + _page_statistics.skip_page_header_num += _state == INITIALIZED; + if constexpr (OFFSET_INDEX) { + if (_offset_index != nullptr) { + if (UNLIKELY(_page_index + 1 >= _offset_index->page_locations.size())) { + return Status::Corruption("Parquet OffsetIndex has no next page location"); + } + _page_index++; + _start_row = _offset_index->page_locations[_page_index].first_row_index; + if (_page_index + 1 < _offset_index->page_locations.size()) { + _end_row = _offset_index->page_locations[_page_index + 1].first_row_index; + } else { + _end_row = _total_rows; + } + int64_t next_page_offset = _offset_index->page_locations[_page_index].offset; + _offset = next_page_offset; + _next_header_offset = next_page_offset; + _state = INITIALIZED; + return Status::OK(); + } + } + + if (UNLIKELY(_offset == _start_offset)) { + return Status::Corruption("should parse first page."); + } + if (is_header_v2()) { + _start_row += _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _start_row += _cur_page_header.data_page_header.num_values; + } + _offset = _next_header_offset; + _state = INITIALIZED; + + return Status::OK(); + } + + Status skip_auxiliary_page() { + if constexpr (OFFSET_INDEX) { + // OffsetIndex enumerates data pages only, so an auxiliary physical page must not + // consume a logical page-location entry while advancing to its payload end. + skip_page_data(); + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + + Status dict_next_page() { + if constexpr (OFFSET_INDEX) { + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + + Status get_page_header(const tparquet::PageHeader** page_header) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::InternalError("Page header not parsed"); + } + *page_header = &_cur_page_header; + return Status::OK(); + } + + Status get_page_data(Slice& slice); + + // Skip page data and update offset (used when data is loaded from cache) + void skip_page_data() { + if (_state == HEADER_PARSED) { + _offset += _cur_page_header.compressed_page_size; + _state = DATA_LOADED; + } + } + + const std::vector& header_bytes() const { return _header_buf; } + // header start offset for current page + int64_t header_start_offset() const { + return static_cast(_next_header_offset) - static_cast(_last_header_size) - + static_cast(_cur_page_header.compressed_page_size); + } + uint64_t file_end_offset() const { return _end_offset; } + bool cached_decompressed() const { + return should_cache_decompressed(&_cur_page_header, _metadata, + _page_read_ctx.data_page_v2_always_compressed); + } + + PageStatistics& page_statistics() { return _page_statistics; } + + bool is_header_v2() { return _cur_page_header.__isset.data_page_header_v2; } + + // Returns whether the current page's cache payload is decompressed + bool is_cache_payload_decompressed() const { return _is_cache_payload_decompressed; } + + size_t start_row() const { return _start_row; } + + size_t end_row() const { return _end_row; } + + bool has_active_offset_index() const { + if constexpr (OFFSET_INDEX) { + return _offset_index != nullptr; + } + return false; + } + + void discard_offset_index() { + if constexpr (OFFSET_INDEX) { + _offset_index = nullptr; + } + } + + // Accessors for cache handle + bool has_page_cache_handle() const { return _page_cache_handle.cache() != nullptr; } + const doris::PageCacheHandle& page_cache_handle() const { return _page_cache_handle; } + StoragePageCache::CacheKey make_page_cache_key(int64_t offset) const { + return _page_cache_key_builder.make_key(_end_offset, offset); + } + +private: + Status _validate_page_header(uint32_t header_size) const; + void _reconcile_offset_index_location(uint64_t header_offset, uint32_t header_size); + void _update_sequential_row_range(); + + enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED }; + PageReaderState _state = INITIALIZED; + PageStatistics _page_statistics; + + io::BufferedStreamReader* _reader = nullptr; + io::IOContext* _io_ctx = nullptr; + // current reader offset in file location. + uint64_t _offset = 0; + // this page offset in file location. + uint64_t _start_offset = 0; + uint64_t _end_offset = 0; + uint64_t _next_header_offset = 0; + // current page row range + size_t _start_row = 0; + size_t _end_row = 0; + // total rows in this column chunk + size_t _total_rows = 0; + // Column metadata for this column chunk + const tparquet::ColumnMetaData& _metadata; + // Session-level parquet page cache options + ParquetPageReadContext _page_read_ctx; + // for page index + size_t _page_index = 0; + const tparquet::OffsetIndex* _offset_index; + + tparquet::PageHeader _cur_page_header; + bool _is_cache_payload_decompressed = true; + + // Page cache members + ParquetPageCacheKeyBuilder _page_cache_key_builder; + doris::PageCacheHandle _page_cache_handle; + // stored header bytes when cache miss so we can insert header+payload into cache + std::vector _header_buf; + // last parsed header size in bytes + uint32_t _last_header_size = 0; +}; + +template +std::unique_ptr> create_page_reader( + io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length, + size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& ctx, const tparquet::OffsetIndex* offset_index = nullptr) { + return std::make_unique>( + reader, io_ctx, offset, length, total_rows, metadata, ctx, offset_index); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp new file mode 100644 index 00000000000000..233b37c096e300 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -0,0 +1,772 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native_column_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "runtime/runtime_state.h" + +namespace doris::format::parquet { +namespace { + +constexpr size_t MAX_RETAINED_BATCH_SCRATCH_BYTES = 4UL << 20; + +DataTypePtr projected_type(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection) { + if (!format::is_partial_projection(projection)) { + return schema.type; + } + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + return schema.type; + case ParquetColumnSchemaKind::STRUCT: { + DataTypes child_types; + Strings child_names; + child_types.reserve(projection->children.size()); + child_names.reserve(projection->children.size()); + for (const auto& child_projection : projection->children) { + const auto child_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(child_it != schema.children.end()); + child_types.push_back(make_nullable(projected_type(**child_it, &child_projection))); + child_names.push_back((*child_it)->name); + } + DataTypePtr type = std::make_shared(child_types, child_names); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto* child_projection = + format::find_child_projection(projection, schema.children[0]->local_id); + DORIS_CHECK(child_projection != nullptr); + DataTypePtr type = std::make_shared( + projected_type(*schema.children[0], child_projection)); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::MAP: { + DORIS_CHECK(schema.children.size() == 2); + const auto* value_projection = + format::find_child_projection(projection, schema.children[1]->local_id); + DORIS_CHECK(value_projection != nullptr); + DataTypePtr type = std::make_shared( + make_nullable(schema.children[0]->type), + make_nullable(projected_type(*schema.children[1], value_projection))); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + } + DORIS_CHECK(false); + return nullptr; +} + +const NativeFieldSchema* find_child_field(const NativeFieldSchema& parent, + const ParquetColumnSchema& child) { + auto field_it = std::ranges::find_if(parent.children, [&](const NativeFieldSchema& field) { + return (child.parquet_field_id >= 0 && field.field_id == child.parquet_field_id) || + field.name == child.name; + }); + return field_it == parent.children.end() ? nullptr : &*field_it; +} + +void collect_physical_subtree_ids(const NativeFieldSchema& field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + ids->insert(field.get_column_id()); + for (const auto& child : field.children) { + collect_physical_subtree_ids(child, ids); + } +} + +void collect_projected_ids(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const NativeFieldSchema& native_field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + if (!format::is_partial_projection(projection)) { + return; + } + for (const auto& child_projection : projection->children) { + const auto schema_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(schema_it != schema.children.end()); + const NativeFieldSchema* child_field = find_child_field(native_field, **schema_it); + DORIS_CHECK(child_field != nullptr); + if (format::is_full_projection(&child_projection)) { + // A full child path is a request for its complete physical subtree. Keeping only the + // child group id makes its grandchildren SkipReadingReaders and silently defaults + // STRUCT fields (or breaks ARRAY/MAP shape invariants). + collect_physical_subtree_ids(*child_field, ids); + } else { + ids->insert(child_field->get_column_id()); + collect_projected_ids(**schema_it, &child_projection, *child_field, ids); + } + } + if (schema.kind == ParquetColumnSchemaKind::MAP) { + DORIS_CHECK(!native_field.children.empty()); + // MAP entry existence and offsets are owned by the key stream even for value-only + // projections. Keep the key reader live so the native complex reader can validate + // key/value entry alignment while constructing offsets. + ids->insert(native_field.children[0].get_column_id()); + } +} + +Status append_non_null_dictionary_values(MutableColumnPtr& target, MutableColumnPtr values) { + DORIS_CHECK(target); + DORIS_CHECK(values); + const size_t value_count = values->size(); + if (auto* nullable = check_and_get_column(*target); nullable != nullptr) { + nullable->get_nested_column().insert_range_from(*values, 0, value_count); + auto& null_map = nullable->get_null_map_data(); + null_map.resize_fill(null_map.size() + value_count, 0); + return Status::OK(); + } + target->insert_range_from(*values, 0, value_count); + return Status::OK(); +} + +} // namespace + +NativeColumnReader::NativeColumnReader(const ParquetColumnSchema& schema, + DataTypePtr projected_type, + ParquetColumnReaderProfile profile) + : ParquetColumnReader(schema, std::move(projected_type), profile), + _nested(schema.kind != ParquetColumnSchemaKind::PRIMITIVE) {} + +NativeColumnReader::~NativeColumnReader() { + (void)sync_native_profile(); +} + +Status NativeColumnReader::create( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter, ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + if (reader == nullptr) { + return Status::InvalidArgument("Native parquet reader result is null"); + } + if (file == nullptr || metadata == nullptr) { + return Status::InvalidArgument("Native parquet file context is not initialized"); + } + if (row_group_id < 0 || + row_group_id >= static_cast(metadata->to_thrift().row_groups.size())) { + return Status::InvalidArgument("Invalid native parquet row group {}", row_group_id); + } + const auto& native_schema = metadata->schema(); + if (column_schema.local_id < 0 || column_schema.local_id >= native_schema.size()) { + return Status::InvalidArgument("Invalid native parquet top-level column id {} for {}", + column_schema.local_id, column_schema.name); + } + auto* field = const_cast(native_schema.get_column(column_schema.local_id)); + DORIS_CHECK(field != nullptr); + if (field->name != column_schema.name && + !(field->field_id >= 0 && field->field_id == column_schema.parquet_field_id)) { + return Status::Corruption( + "Native/metadata parquet schema mismatch at column {}: native={}, arrow={}", + column_schema.local_id, field->name, column_schema.name); + } + + auto type = projected_type(column_schema, projection); + std::shared_ptr schema_node; + RETURN_IF_ERROR(build_native_schema_node(type, column_schema, &schema_node)); + std::set projected_ids; + collect_projected_ids(column_schema, projection, *field, &projected_ids); + + auto native_reader = std::unique_ptr( + new NativeColumnReader(column_schema, std::move(type), profile)); + RETURN_IF_ERROR(native_reader->init( + std::move(file), metadata, row_group_id, field, std::move(schema_node), + std::move(projected_ids), selected_ranges, offset_indexes, timezone, io_ctx, + runtime_state, enable_page_cache, page_cache_file_key, enable_dictionary_filter)); + *reader = std::move(native_reader); + return Status::OK(); +} + +Status NativeColumnReader::init( + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + NativeFieldSchema* field, std::shared_ptr schema_node, + std::set projected_column_ids, const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(field != nullptr); + DORIS_CHECK(schema_node != nullptr); + const auto& row_group = metadata->to_thrift().row_groups[row_group_id]; + DORIS_CHECK(row_group.num_rows > 0); + _row_group_rows = row_group.num_rows; + _selected_ranges = selected_ranges; + DORIS_CHECK(!_selected_ranges.empty()); + for (const auto& range : _selected_ranges) { + DORIS_CHECK(range.start >= 0); + DORIS_CHECK(range.length > 0); + DORIS_CHECK(range.start + range.length <= _row_group_rows); + _row_ranges.add(segment_v2::RowRange(range.start, range.start + range.length)); + } + // Offset indexes are immutable row-group metadata owned by ParquetScanScheduler. Sharing them + // avoids retaining the full N-column page-location map once per projected reader. + _offset_indexes = &offset_indexes; + _schema_node = std::move(schema_node); + _projected_column_ids = std::move(projected_column_ids); + _dictionary_filter_enabled = enable_dictionary_filter; + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + const size_t max_buffer_size = std::min(max_group_buffer, max_column_buffer); + RuntimeState* native_runtime_state = runtime_state; + const bool runtime_page_cache_enabled = + runtime_state == nullptr || + runtime_state->query_options().enable_parquet_file_page_cache; + if (runtime_page_cache_enabled != enable_page_cache) { + TQueryOptions query_options = + runtime_state == nullptr ? TQueryOptions() : runtime_state->query_options(); + query_options.__set_enable_parquet_file_page_cache(enable_page_cache); + _page_cache_runtime_state = RuntimeState::create_unique(query_options, TQueryGlobals()); + native_runtime_state = _page_cache_runtime_state.get(); + } + const auto& thrift_metadata = metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + RETURN_IF_ERROR(native::ColumnReader::create( + std::move(file), field, row_group, _row_ranges, timezone, io_ctx, _native_reader, + max_buffer_size, *_offset_indexes, native_runtime_state, false, _projected_column_ids, + _filter_column_ids, page_cache_file_key, compat, + runtime_state != nullptr && runtime_state->enable_strict_mode())); + DORIS_CHECK(_native_reader != nullptr); + _skip_column = _type->create_column(); + return Status::OK(); +} + +Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, MutableColumnPtr& column, + const DataTypePtr& output_type, bool dictionary_ids, + int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(column); + DORIS_CHECK(output_type != nullptr); + DORIS_CHECK(rows_read != nullptr); + *rows_read = 0; + if (rows == 0) { + return Status::OK(); + } + + native::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + ColumnPtr native_column(std::move(column)); + bool eof = false; + int64_t native_calls = 0; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + ++native_calls; + size_t loop_rows = 0; + RETURN_IF_ERROR(_native_reader->read_column_data( + native_column, output_type, _schema_node, filter, + static_cast(rows - *rows_read), &loop_rows, &eof, dictionary_ids)); + if (loop_rows == 0 && !eof) { + // A selected RowRanges plan may reject the current data page completely. V1 advances + // the page cursor and deliberately returns zero rows so the caller can request the + // next page. Bound consecutive empty transitions by the Row Group row count to retain + // a deterministic corruption exit if a decoder ever stops advancing. + if (++consecutive_empty_calls > _row_group_rows + 1) { + column = IColumn::mutate(std::move(native_column)); + return Status::Corruption("Native parquet reader made no progress for column {}", + _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + column = IColumn::mutate(std::move(native_column)); + if (_profile.native_read_calls != nullptr) { + COUNTER_UPDATE(_profile.native_read_calls, native_calls); + } + if (_nested && _profile.nested_batches != nullptr) { + COUNTER_UPDATE(_profile.nested_batches, 1); + } + release_batch_scratch_if_needed(); + if (*rows_read != rows) { + return Status::Corruption("Native parquet reader returned {} rows, expected {} for {}", + *rows_read, rows, _name); + } + return Status::OK(); +} + +Status NativeColumnReader::read_with_fixed_width_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, + const VExprSPtrs& conjuncts, int column_id, + IColumn* projected_column, + IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(rows_read != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *rows_read = 0; + *used_filter = false; + if (rows == 0) { + return Status::OK(); + } + + native::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + bool eof = false; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + size_t loop_rows = 0; + IColumn::Filter loop_filter; + bool loop_used = false; + RETURN_IF_ERROR(_native_reader->read_fixed_width_filter( + conjuncts, column_id, filter, static_cast(rows - *rows_read), + projected_column, &loop_filter, &loop_rows, &eof, &loop_used)); + if (!loop_used) { + if (UNLIKELY(*rows_read != 0)) { + // Footer encoding lists are untrusted. Once a prior page advanced the cursor, a + // typed fallback would restart the request at the wrong row, so reject the file + // instead of terminating the BE or returning shifted results. + return Status::Corruption( + "Parquet fixed-width predicate encoding changed after {} rows for column " + "{}", + *rows_read, _name); + } + row_filter->clear(); + return Status::OK(); + } + row_filter->insert(row_filter->end(), loop_filter.begin(), loop_filter.end()); + if (loop_rows == 0 && !eof) { + if (++consecutive_empty_calls > _row_group_rows + 1) { + return Status::Corruption( + "Native parquet fixed-width predicate made no progress for column {}", + _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + if (*rows_read != rows) { + return Status::Corruption( + "Native parquet fixed-width predicate returned {} rows, expected {} for {}", + *rows_read, rows, _name); + } + *used_filter = true; + release_batch_scratch_if_needed(); + return Status::OK(); +} + +void NativeColumnReader::release_batch_scratch_if_needed() { + // PLAIN predicate batches bypass materialization but share the same persistent decoder tree, + // so both read paths must advance the retained-capacity aging clock. + constexpr size_t SCRATCH_CHECK_BATCH_INTERVAL = 16; + if (++_batches_since_scratch_check >= SCRATCH_CHECK_BATCH_INTERVAL) { + _native_reader->release_batch_scratch(MAX_RETAINED_BATCH_SCRATCH_BYTES); + _batches_since_scratch_check = 0; + } +} + +Status NativeColumnReader::validate_selected_span(int64_t rows) { + DORIS_CHECK(rows >= 0); + while (_selected_range_idx < _selected_ranges.size()) { + const auto& range = _selected_ranges[_selected_range_idx]; + const int64_t range_end = range.start + range.length; + if (_logical_row_position < range_end) { + break; + } + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + return Status::Corruption("Native parquet read past selected ranges for column {}", _name); + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start || + rows > range.start + range.length - _logical_row_position) { + return Status::Corruption( + "Native parquet read [{}, {}) crosses selected range [{}, {}) for column {}", + _logical_row_position, _logical_row_position + rows, range.start, + range.start + range.length, _name); + } + return Status::OK(); +} + +void NativeColumnReader::advance_selected_span(int64_t rows) { + _logical_row_position += rows; + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } +} + +Status NativeColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + RETURN_IF_ERROR(validate_selected_span(rows)); + RETURN_IF_ERROR(read_with_filter(rows, nullptr, false, column, _type, false, rows_read)); + advance_selected_span(*rows_read); + update_reader_read_rows(*rows_read); + return Status::OK(); +} + +Status NativeColumnReader::skip(int64_t rows) { + if (rows <= 0) { + return Status::OK(); + } + DORIS_CHECK(_logical_row_position <= _row_group_rows - rows); + int64_t remaining = rows; + int64_t native_skipped_rows = 0; + while (remaining > 0) { + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + _logical_row_position += remaining; + break; + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start) { + const int64_t gap = std::min(remaining, range.start - _logical_row_position); + _logical_row_position += gap; + remaining -= gap; + continue; + } + const int64_t selected_rows = detail::bounded_native_lazy_skip_rows( + std::min(remaining, range.start + range.length - _logical_row_position)); + _skip_column->clear(); + // Pending skips can span many filtered batches and are replayed for every lazy column. + // Chunking here bounds each dense bitmap while preserving one logical scheduler skip. + _filter_scratch.assign(static_cast(selected_rows), 0); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(selected_rows, _filter_scratch.data(), true, _skip_column, + _type, false, &rows_read)); + DORIS_CHECK(_skip_column->empty()); + DORIS_CHECK(rows_read == selected_rows); + _logical_row_position += rows_read; + native_skipped_rows += rows_read; + remaining -= rows_read; + } + update_reader_skip_rows(native_skipped_rows); + return Status::OK(); +} + +Status NativeColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column) { + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + const size_t old_size = column->size(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, column, _type, + false, &rows_read)); + advance_selected_span(rows_read); + if (column->size() != old_size + selected_rows) { + return Status::Corruption( + "Native parquet selection appended {} rows, expected {} for column {}", + column->size() - old_size, selected_rows, _name); + } + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + return Status::OK(); +} + +Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, + IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + *used_filter = false; + row_filter->clear(); + if (!_dictionary_filter_enabled) { + return Status::OK(); + } + *used_filter = true; + + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + const bool nullable = _type->is_nullable(); + DataTypePtr id_type = std::make_shared(); + if (nullable) { + id_type = make_nullable(id_type); + } + if (!_dictionary_id_column) { + _dictionary_id_column = id_type->create_column(); + } + _dictionary_id_column->clear(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, + _dictionary_id_column, id_type, true, &rows_read)); + advance_selected_span(rows_read); + if (_dictionary_id_column->size() != selected_rows) { + return Status::Corruption( + "Native parquet dictionary reader appended {} rows, expected {} for {}", + _dictionary_id_column->size(), selected_rows, _name); + } + + const ColumnInt32* ids = nullptr; + const NullMap* null_map = nullptr; + if (const auto* nullable_ids = check_and_get_column(*_dictionary_id_column); + nullable_ids != nullptr) { + ids = check_and_get_column(nullable_ids->get_nested_column()); + null_map = &nullable_ids->get_null_map_data(); + } else { + ids = check_and_get_column(*_dictionary_id_column); + } + DORIS_CHECK(ids != nullptr); + + if (!_matched_dictionary_ids) { + _matched_dictionary_ids = ColumnInt32::create(); + } + _matched_dictionary_ids->clear(); + auto& matched_ids = assert_cast(*_matched_dictionary_ids).get_data(); + row_filter->reserve(selected_rows); + const auto& id_data = ids->get_data(); + for (size_t row = 0; row < selected_rows; ++row) { + bool keep = false; + if (null_map == nullptr || (*null_map)[row] == 0) { + const int32_t dictionary_id = id_data[row]; + if (dictionary_id < 0 || + static_cast(dictionary_id) >= dictionary_filter.size()) { + return Status::Corruption( + "Invalid parquet dictionary id {} for column {} with {} entries", + dictionary_id, _name, dictionary_filter.size()); + } + keep = dictionary_filter[static_cast(dictionary_id)] != 0; + if (keep) { + matched_ids.push_back(dictionary_id); + } + } + row_filter->push_back(keep ? 1 : 0); + } + + const auto* matched_id_column = check_and_get_column(*_matched_dictionary_ids); + DORIS_CHECK(matched_id_column != nullptr); + auto matched_values = + DORIS_TRY(_native_reader->materialize_dictionary_values(matched_id_column, _type)); + RETURN_IF_ERROR(append_non_null_dictionary_values(column, std::move(matched_values))); + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(cast_set(matched_ids.size())); + update_reader_skip_rows(batch_rows - cast_set(matched_ids.size())); + return Status::OK(); +} + +Status NativeColumnReader::select_with_fixed_width_filter( + const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, + const VExprSPtrs& conjuncts, int column_id, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_fixed_width_filter(batch_rows, filter_data, selected_rows == 0, + conjuncts, column_id, projected_column, row_filter, + &rows_read, used_filter)); + if (!*used_filter) { + return Status::OK(); + } + DORIS_CHECK_EQ(rows_read, batch_rows); + if (row_filter->size() != selected_rows) { + return Status::Corruption( + "Native parquet fixed-width predicate returned {} selected rows, expected {} for " + "{}", + row_filter->size(), selected_rows, _name); + } + advance_selected_span(rows_read); + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + return Status::OK(); +} + +void NativeColumnReader::flush_profile() { + record_page_fragments(sync_native_profile()); +} + +bool NativeColumnReader::crossed_page_since_last_batch() { + if (_native_reader == nullptr) { + return false; + } + const auto stats = _native_reader->column_statistics(); + bool crossed_page = false; + if (stats.leaf_page_read_counters.size() == _batch_leaf_page_read_counters.size()) { + for (size_t leaf = 0; leaf < stats.leaf_page_read_counters.size(); ++leaf) { + crossed_page |= + stats.leaf_page_read_counters[leaf] - _batch_leaf_page_read_counters[leaf] > 1; + } + } else { + // The first snapshot covers the first batch; later snapshots must keep the stable tree shape. + for (const int64_t page_reads : stats.leaf_page_read_counters) { + crossed_page |= page_reads > 1; + } + } + _batch_leaf_page_read_counters = stats.leaf_page_read_counters; + return crossed_page; +} + +Result NativeColumnReader::dictionary_values() { + DORIS_CHECK(_native_reader != nullptr); + return _native_reader->dictionary_values(_type); +} + +void NativeColumnReader::record_page_fragments(int64_t page_fragments) { + if (_profile.native_page_fragments != nullptr) { + COUNTER_UPDATE(_profile.native_page_fragments, page_fragments); + } +} + +int64_t NativeColumnReader::sync_native_profile() { + if (_native_reader == nullptr) { + return 0; + } + const auto stats = _native_reader->column_statistics(); + const auto& reported = _reported_native_stats; + if (_profile.decompress_time != nullptr) { + COUNTER_UPDATE(_profile.decompress_time, stats.decompress_time - reported.decompress_time); + } + if (_profile.decompress_count != nullptr) { + COUNTER_UPDATE(_profile.decompress_count, stats.decompress_cnt - reported.decompress_cnt); + } + if (_profile.decode_header_time != nullptr) { + COUNTER_UPDATE(_profile.decode_header_time, + stats.decode_header_time - reported.decode_header_time); + } + if (_profile.decode_value_time != nullptr) { + COUNTER_UPDATE(_profile.decode_value_time, + stats.decode_value_time - reported.decode_value_time); + } + if (_profile.decode_dictionary_time != nullptr) { + COUNTER_UPDATE(_profile.decode_dictionary_time, + stats.decode_dict_time - reported.decode_dict_time); + } + if (_profile.decode_level_time != nullptr) { + COUNTER_UPDATE(_profile.decode_level_time, + stats.decode_level_time - reported.decode_level_time); + } + if (_profile.decode_null_map_time != nullptr) { + COUNTER_UPDATE(_profile.decode_null_map_time, + stats.decode_null_map_time - reported.decode_null_map_time); + } + if (_profile.materialization_time != nullptr) { + COUNTER_UPDATE(_profile.materialization_time, + stats.materialization_time - reported.materialization_time); + } + if (_profile.hybrid_selection_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_batches, + stats.hybrid_selection_batches - reported.hybrid_selection_batches); + } + if (_profile.hybrid_selection_ranges != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_ranges, + stats.hybrid_selection_ranges - reported.hybrid_selection_ranges); + } + if (_profile.hybrid_selection_null_fallback_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_null_fallback_batches, + stats.hybrid_selection_null_fallback_batches - + reported.hybrid_selection_null_fallback_batches); + } + if (_profile.page_index_read_calls != nullptr) { + COUNTER_UPDATE(_profile.page_index_read_calls, + stats.page_index_read_calls - reported.page_index_read_calls); + } + if (_profile.skip_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.skip_page_header_count, + stats.skip_page_header_num - reported.skip_page_header_num); + } + if (_profile.parse_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.parse_page_header_count, + stats.parse_page_header_num - reported.parse_page_header_num); + } + if (_profile.read_page_header_time != nullptr) { + COUNTER_UPDATE(_profile.read_page_header_time, + stats.read_page_header_time - reported.read_page_header_time); + } + const int64_t page_read_delta = stats.page_read_counter - reported.page_read_counter; + if (_profile.page_read_count != nullptr) { + COUNTER_UPDATE(_profile.page_read_count, page_read_delta); + } + if (_profile.page_cache_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_write_count, + stats.page_cache_write_counter - reported.page_cache_write_counter); + } + if (_profile.page_cache_compressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_write_count, + stats.page_cache_compressed_write_counter - + reported.page_cache_compressed_write_counter); + } + if (_profile.page_cache_decompressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_write_count, + stats.page_cache_decompressed_write_counter - + reported.page_cache_decompressed_write_counter); + } + if (_profile.page_cache_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_hit_count, + stats.page_cache_hit_counter - reported.page_cache_hit_counter); + } + if (_profile.page_cache_miss_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_miss_count, + stats.page_cache_missing_counter - reported.page_cache_missing_counter); + } + if (_profile.page_cache_compressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_hit_count, + stats.page_cache_compressed_hit_counter - + reported.page_cache_compressed_hit_counter); + } + if (_profile.page_cache_decompressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_hit_count, + stats.page_cache_decompressed_hit_counter - + reported.page_cache_decompressed_hit_counter); + } + _reported_native_stats = stats; + return page_read_delta; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h new file mode 100644 index 00000000000000..1d92735bfa19d3 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format_v2/column_data.h" +#include "format_v2/parquet/native_schema_node.h" +#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/native/column_reader.h" + +namespace doris { +class RuntimeState; +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet { + +class NativeParquetMetadata; + +namespace detail { +inline constexpr int64_t MAX_NATIVE_LAZY_SKIP_ROWS = std::numeric_limits::max(); + +inline int64_t bounded_native_lazy_skip_rows(int64_t rows) { + return std::min(rows, MAX_NATIVE_LAZY_SKIP_ROWS); +} +} // namespace detail + +// Production adapter from FileScannerV2's selection-oriented reader contract to Doris' native +// Parquet page/encoding reader. The owned native reader decodes page bytes directly into the final +// Doris column. It never creates an Arrow Array/Builder, DecodedColumnView, or intermediate nested +// values_column. +// +// Cursor contract: +// - read(rows) consumes and appends exactly `rows` logical top-level rows; +// - select(selection, batch_rows) consumes `batch_rows` and appends only selected rows; +// - skip(rows) consumes `rows` with an all-false FilterMap and appends no payload; +// - one adapter lives for one top-level column in one Row Group, so decoder dictionaries, +// decompression buffers, level buffers, converters, and destination capacity survive adaptive +// batch-size changes. +class NativeColumnReader final : public ParquetColumnReader { +public: + static Status create(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, io::FileReaderSPtr file, + const NativeParquetMetadata* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, + RuntimeState* runtime_state, bool enable_page_cache, + const std::string& page_cache_file_key, bool enable_dictionary_filter, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + ~NativeColumnReader() override; + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + Status select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, + MutableColumnPtr& column) override; + Status select_with_dictionary_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, IColumn::Filter* row_filter, + bool* used_filter) override; + Status select_with_fixed_width_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, const VExprSPtrs& conjuncts, + int column_id, IColumn* projected_column, + IColumn::Filter* row_filter, bool* used_filter) override; + void flush_profile() override; + bool crossed_page_since_last_batch() override; + Result dictionary_values() override; + +private: + NativeColumnReader(const ParquetColumnSchema& schema, DataTypePtr projected_type, + ParquetColumnReaderProfile profile); + + Status init(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + NativeFieldSchema* field, std::shared_ptr schema_node, + std::set projected_column_ids, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter); + + Status read_with_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + MutableColumnPtr& column, const DataTypePtr& output_type, + bool dictionary_ids, int64_t* rows_read); + Status read_with_fixed_width_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + const VExprSPtrs& conjuncts, int column_id, + IColumn* projected_column, IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter); + void release_batch_scratch_if_needed(); + int64_t sync_native_profile(); + void record_page_fragments(int64_t page_fragments); + Status validate_selected_span(int64_t rows); + void advance_selected_span(int64_t rows); + + // Native ParquetColumnReader keeps a reference to RowRanges; declare it before the reader. + segment_v2::RowRanges _row_ranges; + std::set _projected_column_ids; + std::set _filter_column_ids; + const std::unordered_map* _offset_indexes = nullptr; + std::shared_ptr _schema_node; + std::unique_ptr _native_reader; + std::unique_ptr _page_cache_runtime_state; + std::vector _selected_ranges; + size_t _selected_range_idx = 0; + int64_t _logical_row_position = 0; + int64_t _row_group_rows = 0; + + bool _dictionary_filter_enabled = false; + bool _nested = false; + // The native tree exposes cumulative statistics. Keep the last reported snapshot so each + // FileScannerV2 batch contributes only its delta to RuntimeProfile. + native::ColumnReader::ColumnStatistics _reported_native_stats; + // Page-crossing is sampled at every scheduler batch, independently of amortized profile flushes. + std::vector _batch_leaf_page_read_counters; + std::vector _filter_scratch; + size_t _batches_since_scratch_check = 0; + MutableColumnPtr _skip_column; + MutableColumnPtr _dictionary_id_column; + MutableColumnPtr _matched_dictionary_ids; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp b/be/src/format_v2/parquet/reader/nested_column_materializer.cpp deleted file mode 100644 index e06b7eaaf317e7..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/nested_column_materializer.h" - -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" - -namespace doris::format::parquet { - -ColumnArray* array_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnMap* map_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return &nullable_column->get_null_map_data(); - } - return nullptr; -} - -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts) { - offsets.reserve(offsets.size() + entry_counts.size()); - uint64_t current_offset = offsets.empty() ? 0 : offsets.back(); - for (const auto entry_count : entry_counts) { - current_offset += entry_count; - offsets.push_back(current_offset); - } -} - -void append_parent_nulls(NullMap* dst, const NullMap& src) { - if (dst == nullptr) { - return; // target column is not nullable; no null marker is needed - } - dst->insert(src.begin(), src.end()); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.h b/be/src/format_v2/parquet/reader/nested_column_materializer.h deleted file mode 100644 index 90fac01eb2f5e5..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.h +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "core/column/column.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_struct.h" - -namespace doris::format::parquet { - -// ============================================================================ -// ============================================================================ - -ColumnArray* array_column_from_output(MutableColumnPtr& column); - -ColumnMap* map_column_from_output(MutableColumnPtr& column); - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column); - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column); - -// offsets[i] = offsets[i-1] + entry_counts[i]. -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts); - -void append_parent_nulls(NullMap* dst, const NullMap& src); - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp deleted file mode 100644 index fd261ef5219d27..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp +++ /dev/null @@ -1,803 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "core/string_ref.h" -#include "runtime/runtime_profile.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -DecodedTimeUnit decoded_time_unit(ParquetTimeUnit time_unit) { - switch (time_unit) { - case ParquetTimeUnit::MILLIS: - return DecodedTimeUnit::MILLIS; - case ParquetTimeUnit::MICROS: - return DecodedTimeUnit::MICROS; - case ParquetTimeUnit::NANOS: - return DecodedTimeUnit::NANOS; - case ParquetTimeUnit::UNKNOWN: - default: - return DecodedTimeUnit::UNKNOWN; - } -} - -Status decoded_fixed_value_size(const std::string& column_name, DecodedValueKind value_kind, - size_t* value_size) { - switch (value_kind) { - case DecodedValueKind::BOOL: - *value_size = sizeof(bool); - return Status::OK(); - case DecodedValueKind::INT32: - *value_size = sizeof(int32_t); - return Status::OK(); - case DecodedValueKind::UINT32: - *value_size = sizeof(uint32_t); - return Status::OK(); - case DecodedValueKind::INT64: - *value_size = sizeof(int64_t); - return Status::OK(); - case DecodedValueKind::UINT64: - *value_size = sizeof(uint64_t); - return Status::OK(); - case DecodedValueKind::INT96: - *value_size = 12; - return Status::OK(); - case DecodedValueKind::FLOAT: - *value_size = sizeof(float); - return Status::OK(); - case DecodedValueKind::DOUBLE: - *value_size = sizeof(double); - return Status::OK(); - case DecodedValueKind::BINARY: - case DecodedValueKind::FIXED_BINARY: - return Status::InvalidArgument("Parquet binary value kind has no fixed value size for {}", - column_name); - } - return Status::InternalError("Unknown decoded value kind for column {}", column_name); -} - -Status get_binary_chunks(const std::string& column_name, - ::parquet::internal::RecordReader& record_reader, - std::vector>* chunks) { - if (auto* dictionary_reader = - dynamic_cast<::parquet::internal::DictionaryRecordReader*>(&record_reader); - dictionary_reader != nullptr) { - auto chunked = dictionary_reader->GetResult(); - if (chunked == nullptr) { - return Status::Corruption( - "Parquet dictionary record reader returned null result for column {}", - column_name); - } - *chunks = chunked->chunks(); - return Status::OK(); - } - auto* binary_reader = dynamic_cast<::parquet::internal::BinaryRecordReader*>(&record_reader); - if (binary_reader == nullptr) { - return Status::InternalError("Parquet binary record reader is not available for column {}", - column_name); - } - *chunks = binary_reader->GetBuilderChunks(); - return Status::OK(); -} - -Status append_dictionary_binary_values(const std::string& column_name, - const ::arrow::DictionaryArray& dictionary_array, - std::vector* values) { - DORIS_CHECK(values != nullptr); - const auto& dictionary = dictionary_array.dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - column_name); - } - auto append_value = [&](int64_t dictionary_index) -> Status { - if (dictionary_index < 0 || dictionary_index >= dictionary->length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(dictionary.get())) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(dictionary.get())) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back( - reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); - }; - for (int64_t row_idx = 0; row_idx < dictionary_array.length(); ++row_idx) { - if (dictionary_array.IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - RETURN_IF_ERROR(append_value(dictionary_array.GetValueIndex(row_idx))); - } - return Status::OK(); -} - -Status build_binary_values(const std::string& column_name, - const std::vector>& chunks, - int64_t records_read, const NullMap* null_map, - bool read_dense_for_nullable, std::vector* binary_values) { - std::vector compact_values; - auto* values = read_dense_for_nullable ? &compact_values : binary_values; - values->reserve(records_read); - for (const auto& chunk : chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < binary_array->length(); ++row_idx) { - if (binary_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(row_idx, &length); - values->emplace_back(reinterpret_cast(value), length); - } - } else if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < fixed_array->length(); ++row_idx) { - if (fixed_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(row_idx)), - fixed_array->byte_width()); - } - } else if (auto* dictionary_array = dynamic_cast<::arrow::DictionaryArray*>(chunk.get())) { - RETURN_IF_ERROR( - append_dictionary_binary_values(column_name, *dictionary_array, values)); - } else { - return Status::InternalError("Unexpected Arrow binary array type for column {}", - column_name); - } - } - if (read_dense_for_nullable) { - if (null_map == nullptr || null_map->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid dense nullable parquet null map for column {}: rows={}, null_map={}", - column_name, records_read, null_map == nullptr ? 0 : null_map->size()); - } - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (compact_values.size() != static_cast(non_null_count)) { - return Status::Corruption( - "Invalid dense nullable parquet binary values for column {}: values={}, " - "records={}, nulls={}", - column_name, compact_values.size(), records_read, - records_read - non_null_count); - } - binary_values->reserve(records_read); - size_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - binary_values->emplace_back(static_cast(nullptr), 0); - continue; - } - binary_values->emplace_back(compact_values[value_idx++]); - } - return Status::OK(); - } - if (binary_values->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid parquet binary record read result for column {}: rows={}, records={}", - column_name, binary_values->size(), records_read); - } - return Status::OK(); -} - -float half_to_float(uint16_t value) { - const uint32_t sign = (value & 0x8000U) << 16; - const uint32_t exponent = (value & 0x7C00U) >> 10; - const uint32_t mantissa = value & 0x03FFU; - - if (exponent == 0) { - if (mantissa == 0) { - return std::bit_cast(sign); - } - const float subnormal = std::ldexp(static_cast(mantissa), -24); - return sign == 0 ? subnormal : -subnormal; - } - if (exponent == 0x1FU) { - return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); - } - return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); -} - -Status build_float16_values(const std::string& column_name, - const ParquetTypeDescriptor& type_descriptor, - const std::vector& binary_values, int64_t row_count, - std::vector* float_values) { - if (type_descriptor.fixed_length != 2) { - return Status::Corruption("Invalid parquet Float16 length for column {}: {}", column_name, - type_descriptor.fixed_length); - } - if (binary_values.size() != static_cast(row_count)) { - return Status::Corruption( - "Invalid parquet Float16 value count for column {}: values={}, rows={}", - column_name, binary_values.size(), row_count); - } - float_values->resize(static_cast(row_count)); - for (int64_t row = 0; row < row_count; ++row) { - const auto& binary_value = binary_values[static_cast(row)]; - if (binary_value.data == nullptr && binary_value.size == 0) { - (*float_values)[static_cast(row)] = 0; - continue; - } - if (binary_value.data == nullptr || binary_value.size != 2) { - return Status::Corruption( - "Invalid parquet Float16 value for column {} at row {}: data={}, size={}", - column_name, row, binary_value.data == nullptr ? "null" : "non-null", - binary_value.size); - } - uint16_t raw_value = 0; - std::memcpy(&raw_value, binary_value.data, sizeof(raw_value)); - (*float_values)[static_cast(row)] = half_to_float(raw_value); - } - return Status::OK(); -} - -} // namespace - -Status ParquetLeafReader::collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - batch->_values_written = record_reader.values_written(); - - if (!batch->is_binary_value()) { - batch->_fixed_values = record_reader.values(); - return Status::OK(); - } - - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &batch->_binary_chunks)); - batch->_values_written = 0; - for (const auto& chunk : batch->_binary_chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", _name); - } - batch->_values_written += chunk->length(); - } - return Status::OK(); -} - -Status ParquetLeafReader::collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - - // Arrow's RecordReader::Reset() does not reset ByteArray/FLBA builders. GetBuilderChunks() - // (or DictionaryRecordReader::GetResult()) is the documented reset operation and must be - // called before the next ReadRecords(). Otherwise a levels-only skip followed by a normal read - // observes values from both batches; for example, skipping ARRAY ["a", "b"] and then - // reading ["c"] would report one current level but three values. Release the chunks here and - // let the temporary vector destroy them immediately. We deliberately do not inspect or copy - // their payload into a Doris Column, so the levels-only contract still avoids Doris-side value - // materialization. - if (batch->is_binary_value()) { - std::vector> discarded_chunks; - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &discarded_chunks)); - } - - // COUNT(col) and nested skip only need top-level shape. Fixed-width values remain owned by the - // RecordReader and are cleared by Reset(); binary values were released above solely to reset - // the Arrow builder. - batch->_values_written = 0; - return Status::OK(); -} - -// - FLOAT16: binary -> half_to_float -> float_values -Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, MutableColumnPtr& column) const { - std::vector binary_values; - std::vector spaced_values; - std::vector float_values; - DecodedColumnView view; - view.value_kind = batch._value_kind; - view.time_unit = decoded_time_unit(_type_descriptor.time_unit); - view.row_count = row_count; - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.decimal_precision = _type_descriptor.decimal_precision; - view.decimal_scale = _type_descriptor.decimal_scale; - view.fixed_length = _type_descriptor.fixed_length; - view.timestamp_is_adjusted_to_utc = _type_descriptor.timestamp_is_adjusted_to_utc; - view.timezone = _timezone; - view.enable_strict_mode = _enable_strict_mode; - view.null_map = null_map == nullptr || null_map->empty() ? nullptr : null_map->data(); - const bool read_dense_for_nullable = batch._read_dense_for_nullable && view.null_map != nullptr; - - if (_type_descriptor.extra_type_info == ParquetExtraTypeInfo::FLOAT16) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - RETURN_IF_ERROR(build_float16_values(_name, _type_descriptor, binary_values, row_count, - &float_values)); - view.value_kind = DecodedValueKind::FLOAT; - view.values = reinterpret_cast(float_values.data()); - } else if (batch.is_binary_value()) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - view.binary_values = &binary_values; - } else if (read_dense_for_nullable) { - RETURN_IF_ERROR(build_spaced_fixed_values(batch, row_count, null_map, &spaced_values)); - view.values = spaced_values.data(); - } else { - view.values = batch._fixed_values; - } - - if (_decoded_value_appender != nullptr) { - return _decoded_value_appender(column, view); - } - - { - SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& tmp_null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = tmp_null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - tmp_null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(tmp_null_map.data() + old_null_map_size, 0, - tmp_null_map.size() - old_null_map_size); - } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); - } - } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); - } - } - return Status::OK(); -} - -bool ParquetLeafBatch::is_binary_value() const { - return _value_kind == DecodedValueKind::BINARY || _value_kind == DecodedValueKind::FIXED_BINARY; -} - -Status ParquetLeafReader::build_spaced_fixed_values(const ParquetLeafBatch& batch, - int64_t row_count, const NullMap* null_map, - std::vector* spaced_values) const { - DORIS_CHECK(null_map != nullptr); - DORIS_CHECK(spaced_values != nullptr); - size_t value_size = 0; - RETURN_IF_ERROR(decoded_fixed_value_size(_name, batch._value_kind, &value_size)); - spaced_values->resize(static_cast(row_count) * value_size); - const auto non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (batch._values_written != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet values for column {}: values={}, records={}, " - "nulls={}", - _name, batch._values_written, row_count, row_count - non_null_count); - } - auto* dst = spaced_values->data(); - int64_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < row_count; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - continue; // NULL row: skip it and keep the target slot zeroed - } - std::memcpy(dst + static_cast(record_idx) * value_size, - batch._fixed_values + static_cast(value_idx) * value_size, value_size); - ++value_idx; - } - return Status::OK(); -} - -ParquetLeafReader::ParquetLeafReader( - const ::parquet::ColumnDescriptor* descriptor, ParquetTypeDescriptor type_descriptor, - DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile, const cctz::time_zone* timezone, - bool enable_strict_mode, - std::function decoded_value_appender) - : _descriptor(descriptor), - _type_descriptor(type_descriptor), - _type(std::move(type)), - _name(std::move(name)), - _record_reader(std::move(record_reader)), - _profile(profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _decoded_value_appender(std::move(decoded_value_appender)) {} - -Status ParquetLeafReader::read_batch(int64_t batch_rows, ParquetLeafBatch* batch, - int64_t* rows_read) const { - if (batch == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet leaf batch result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - *rows_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet records for column {}: {}", _name, - e.what()); - } - if (*rows_read < 0 || *rows_read > batch_rows) { - return Status::Corruption("Invalid parquet record read result for column {}: {}", _name, - *rows_read); - } - return collect_batch(*_record_reader, batch); -} - -Status ParquetLeafReader::build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const { - if (_descriptor->max_definition_level() == 0) { - return Status::OK(); - } - auto* def_levels = batch.def_levels(); - if (def_levels == nullptr && records_read > 0) { - return Status::Corruption( - "Parquet record reader returned null definition levels for nullable column {}", - _name); - } - const int16_t max_definition_level = _descriptor->max_definition_level(); - null_map->resize(records_read); - auto* __restrict dst = null_map->data(); - const auto* __restrict src = def_levels; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - dst[record_idx] = src[record_idx] != max_definition_level; - } - return Status::OK(); -} - -Status ParquetLeafReader::read_nested_batch(int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - ParquetLeafBatch leaf_batch; - int64_t records_read = 0; - RETURN_IF_ERROR(read_batch(batch_rows, &leaf_batch, &records_read)); - return build_nested_batch_from_leaf_batch(leaf_batch, records_read, value_slot_definition_level, - batch, value_slot_repetition_level); -} - -Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - int64_t records_read = 0; - ParquetLeafBatch leaf_batch; - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - records_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } - if (records_read < 0 || records_read > batch_rows) { - return Status::Corruption("Invalid parquet level read result for column {}: {}", _name, - records_read); - } - RETURN_IF_ERROR(collect_levels_batch(*_record_reader, &leaf_batch)); - return build_nested_levels_batch_from_leaf_batch(leaf_batch, records_read, batch); -} - -Status ParquetLeafReader::build_nested_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar batch is null for column {}", _name); - } - *batch = ParquetNestedScalarBatch(); - batch->value_slot_definition_level = value_slot_definition_level; - batch->value_slot_repetition_level = value_slot_repetition_level; - - batch->records_read = records_read; - if (_type->is_nullable() && leaf_batch.read_dense_for_nullable()) { - return Status::NotSupported( - "Dense nullable parquet nested reader is not supported for column {}", _name); - } - batch->levels_written = leaf_batch.consumed_level_count(); - const int64_t values_written = leaf_batch.values_written(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - if (batch->levels_written == 0 && batch->records_read > 0 && - values_written == batch->records_read && _descriptor->max_definition_level() == 0 && - _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read || values_written < 0 || - values_written > batch->levels_written) { - return Status::Corruption( - "Invalid nested parquet read result for column {}: rows={}, levels={}, values={}", - _name, batch->records_read, batch->levels_written, values_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - - const int16_t leaf_definition_level = _descriptor->max_definition_level(); - // Arrow's RecordReader may emit value placeholders for null ancestors that are below the - // Doris materialization threshold. Those slots must still advance the payload value index; - // otherwise the next defined child level points at the placeholder instead of its real value. - auto count_value_slots = [&](int16_t slot_definition_level) { - int64_t slot_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] >= slot_definition_level && - batch->rep_levels[level_idx] <= value_slot_repetition_level) { - ++slot_count; - } - } - return slot_count; - }; - - const int64_t value_slot_count = count_value_slots(value_slot_definition_level); - int16_t payload_slot_definition_level = value_slot_definition_level; - int64_t payload_value_slot_count = value_slot_count; - while (payload_slot_definition_level > 0 && payload_value_slot_count < values_written) { - --payload_slot_definition_level; - payload_value_slot_count = count_value_slots(payload_slot_definition_level); - } - - int64_t leaf_value_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < value_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - if (batch->def_levels[level_idx] == leaf_definition_level) { - ++leaf_value_count; - } - } - - enum class ValueLayout { LEVELS, VALUE_SLOTS, LEAF_VALUES, PAYLOAD_VALUE_SLOTS }; - ValueLayout value_layout = ValueLayout::LEAF_VALUES; - if (values_written == batch->levels_written) { - value_layout = ValueLayout::LEVELS; - } else if (values_written == value_slot_count) { - value_layout = ValueLayout::VALUE_SLOTS; - } else if (values_written == leaf_value_count) { - value_layout = ValueLayout::LEAF_VALUES; - } else if (values_written == payload_value_slot_count) { - value_layout = ValueLayout::PAYLOAD_VALUE_SLOTS; - } else { - return Status::Corruption( - "Nested parquet reader returned inconsistent value count for column {}: values={}, " - "levels={}, slots={}, leaf_values={}, payload_slots={}, " - "payload_slot_definition_level={}", - _name, values_written, batch->levels_written, value_slot_count, leaf_value_count, - payload_value_slot_count, payload_slot_definition_level); - } - - batch->value_indices.resize(static_cast(batch->levels_written), -1); - NullMap value_nulls(static_cast(values_written), 1); - int64_t value_idx = 0; - const int16_t decoded_slot_definition_level = value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS - ? payload_slot_definition_level - : value_slot_definition_level; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < decoded_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - const bool has_leaf_value = batch->def_levels[level_idx] == leaf_definition_level; - int64_t decoded_value_idx = -1; - if (value_layout == ValueLayout::LEVELS) { - decoded_value_idx = level_idx; - } else if (value_layout == ValueLayout::VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else if (value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else { - if (!has_leaf_value) { - continue; - } - decoded_value_idx = value_idx++; - } - DORIS_CHECK(decoded_value_idx >= 0); - DORIS_CHECK(decoded_value_idx < values_written); - if (has_leaf_value) { - batch->value_indices[static_cast(level_idx)] = decoded_value_idx; - value_nulls[static_cast(decoded_value_idx)] = 0; - } - } - if (value_layout != ValueLayout::LEVELS && value_idx != values_written) { - return Status::Corruption( - "Nested parquet reader value cursor stopped early for column {}: values={}, " - "visited={}", - _name, values_written, value_idx); - } - - const auto value_type = remove_nullable(_type); - batch->values_column = value_type->create_column(); - if (values_written > 0) { - ParquetLeafReader value_reader(_descriptor, _type_descriptor, value_type, _name, - _record_reader, _profile, _timezone, _enable_strict_mode); - RETURN_IF_ERROR(value_reader.append_values(leaf_batch, values_written, &value_nulls, - batch->values_column)); - } - return Status::OK(); -} - -Status ParquetLeafReader::build_nested_levels_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - *batch = ParquetNestedScalarBatch(); - batch->records_read = records_read; - batch->levels_written = leaf_batch.consumed_level_count(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - - // Required flat leaves do not have physical def/rep level buffers. Synthesize one level slot - // per top-level row so the COUNT(col) aggregation code can use the same shape loop. - if (batch->levels_written == 0 && batch->records_read > 0 && - _descriptor->max_definition_level() == 0 && _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read) { - return Status::Corruption( - "Invalid nested parquet levels result for column {}: rows={}, levels={}", _name, - batch->records_read, batch->levels_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - return Status::OK(); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h deleted file mode 100644 index b396b35fd1f32c..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h +++ /dev/null @@ -1,173 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/status.h" -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace arrow { -class Array; -} // namespace arrow - -namespace doris::format::parquet { - -struct ParquetLeafReaderTestAccess; - -// Read result for a nested scalar leaf, separating Dremel-encoded shape from actual values. -// The COUNT(col) aggregation fast path consumes only records_read, levels_written, def_levels, and rep_levels. -// That path does not populate value_indices or values_column, so callers must not call build_nested_column() afterwards. -struct ParquetNestedScalarBatch { - int64_t records_read = 0; - int64_t levels_written = 0; - int16_t value_slot_definition_level = 0; - int16_t value_slot_repetition_level = std::numeric_limits::max(); - std::vector def_levels; - std::vector rep_levels; - std::vector value_indices; - MutableColumnPtr values_column; - - bool empty() const { return levels_written == 0; } -}; - -class ParquetLeafBatch { -public: - int64_t consumed_level_count() const { return _consumed_level_count; } - int64_t decoded_level_count() const { return _decoded_level_count; } - int64_t values_written() const { return _values_written; } - bool read_dense_for_nullable() const { return _read_dense_for_nullable; } - const int16_t* def_levels() const { return _def_levels; } - const int16_t* rep_levels() const { return _rep_levels; } - const std::vector>& binary_chunks() const { - return _binary_chunks; - } - -private: - friend class ParquetLeafReader; - - bool is_binary_value() const; - - DecodedValueKind _value_kind = DecodedValueKind::INT32; - int64_t _consumed_level_count = 0; - int64_t _decoded_level_count = 0; - int64_t _values_written = 0; - const int16_t* _def_levels = nullptr; - const int16_t* _rep_levels = nullptr; - const uint8_t* _fixed_values = nullptr; - bool _read_dense_for_nullable = false; - std::vector> _binary_chunks; -}; - -// read_batch() -> build_null_map() + append_values() -// read_nested_batch() -class ParquetLeafReader { -public: - ParquetLeafReader(const ::parquet::ColumnDescriptor* descriptor, - ParquetTypeDescriptor type_descriptor, DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile = {}, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - std::function - decoded_value_appender = nullptr); - - Status read_batch(int64_t batch_rows, ParquetLeafBatch* batch, int64_t* rows_read) const; - - Status build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const; - - Status append_values(const ParquetLeafBatch& batch, int64_t row_count, const NullMap* null_map, - MutableColumnPtr& column) const; - - // LEVELS / VALUE_SLOTS / LEAF_VALUES / PAYLOAD_VALUE_SLOTS. - Status read_nested_batch( - int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level = std::numeric_limits::max()) const; - - // COUNT(col) and nested-skip shape-only read path. It still calls Arrow - // RecordReader::ReadRecords() to advance the Parquet cursor and obtain def/rep levels, but - // Doris only copies levels: - // - it does not build value_indices or values_column - // - it does not enter DataTypeSerde::read_column_from_decoded_values() - // - for Binary/FLBA, it releases and immediately discards Arrow builder chunks because that is - // the RecordReader's required reset operation; it never copies them into a Doris Column - // This lets COUNT(col) on MAP/ARRAY/STRUCT evaluate top-level NULL state and lets skip advance - // nested shape without Doris-side STRING/BINARY materialization. Arrow RecordReader does not - // expose a public levels-only API, so ReadRecords may still perform required page decoding. - Status read_nested_levels_batch(int64_t batch_rows, ParquetNestedScalarBatch* batch) const; - -private: - friend struct ParquetLeafReaderTestAccess; - - Status collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - // Levels-only variant of collect_batch(). It snapshots only def/rep level state and does not - // expose value buffers. Binary chunks are released only to reset Arrow's builder and are - // immediately discarded. Used by COUNT(col) and nested skip. - Status collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - Status build_spaced_fixed_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, - std::vector* spaced_values) const; - - Status build_nested_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const; - Status build_nested_levels_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - ParquetNestedScalarBatch* batch) const; - - const ::parquet::ColumnDescriptor* _descriptor = - nullptr; // Arrow column descriptor (physical_type, max_dl, max_rl) - ParquetTypeDescriptor - _type_descriptor; // type encoding information (decimal precision, timestamp unit, etc.) - DataTypePtr _type; // Doris target type - std::string _name; // column name for error messages - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader (shared ownership) - ParquetColumnReaderProfile _profile; // profile counters - const cctz::time_zone* _timezone = nullptr; // timezone for timestamp conversion - bool _enable_strict_mode = false; // strict mode for type mismatch errors - std::function _decoded_value_appender; -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp index 4e9a363b13c7cb..dcd601b003ae3c 100644 --- a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp @@ -20,6 +20,7 @@ #include "core/assert_cast.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" +#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" namespace doris::format::parquet { diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp deleted file mode 100644 index 784e4cdc900fb7..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp +++ /dev/null @@ -1,568 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/scalar_column_reader.h" - -#include -#include -#include - -#include -#include -#include - -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -class ParquetNestedScalarValueCursor { -public: - explicit ParquetNestedScalarValueCursor(const ParquetNestedScalarBatch* batch) { reset(batch); } - - void reset(const ParquetNestedScalarBatch* batch) { - DORIS_CHECK(batch != nullptr); - _batch = batch; - } - - Status value_index(const std::string& column_name, int64_t level_idx, int64_t* value_idx) { - DORIS_CHECK(_batch != nullptr); - DORIS_CHECK(value_idx != nullptr); - DORIS_CHECK(level_idx < _batch->levels_written); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(static_cast(level_idx) < _batch->value_indices.size()); - const int64_t computed_value_idx = _batch->value_indices[static_cast(level_idx)]; - if (computed_value_idx < 0) { - return Status::Corruption("Nested parquet value is absent for column {}", column_name); - } - DORIS_CHECK(_batch->values_column.get() != nullptr); - if (computed_value_idx >= _batch->values_column->size()) { - return Status::Corruption("Nested parquet value index is out of range for column {}", - column_name); - } - *value_idx = computed_value_idx; - return Status::OK(); - } - -private: - const ParquetNestedScalarBatch* _batch = nullptr; -}; - -Status append_scalar_batch_value(const ScalarColumnReader& column_reader, - const ParquetNestedScalarBatch& batch, int64_t level_idx, - ParquetNestedScalarValueCursor* value_cursor, - MutableColumnPtr& column) { - DORIS_CHECK(value_cursor != nullptr); - int64_t value_idx = -1; - RETURN_IF_ERROR(value_cursor->value_index(column_reader.name(), level_idx, &value_idx)); - auto* nullable_column = check_and_get_column(*column); - if (nullable_column != nullptr) { - nullable_column->get_nested_column().insert_from(*batch.values_column, - static_cast(value_idx)); - nullable_column->get_null_map_data().push_back(0); - return Status::OK(); - } - column->insert_from(*batch.values_column, static_cast(value_idx)); - return Status::OK(); -} - -Status append_arrow_binary_dictionary_value(const std::string& column_name, - const ::arrow::Array& dictionary, - int64_t dictionary_index, - std::vector* values) { - DORIS_CHECK(values != nullptr); - if (dictionary_index < 0 || dictionary_index >= dictionary.length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast(&dictionary)) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast(&dictionary)) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); -} - -} // namespace - -ScalarColumnReader::ScalarColumnReader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile profile) - : ParquetColumnReader(column_schema, column_schema.type, profile), - _descriptor(column_schema.descriptor), - _type_descriptor(column_schema.type_descriptor), - _record_reader(std::move(record_reader)), - _page_skip_plan(page_skip_plan), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _nested_batch(std::make_unique()) {} - -ScalarColumnReader::~ScalarColumnReader() = default; - -Status ScalarColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet column read result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - auto reader = leaf_reader(); - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(reader.read_batch(rows, &leaf_batch, rows_read)); - - NullMap null_map; - RETURN_IF_ERROR(reader.build_null_map(leaf_batch, *rows_read, &null_map)); - const auto value_kind = decoded_value_kind(_type_descriptor); - const bool is_binary_value = - value_kind == DecodedValueKind::BINARY || value_kind == DecodedValueKind::FIXED_BINARY; - if (!is_binary_value && leaf_batch.read_dense_for_nullable() && !null_map.empty()) { - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map.data()), null_map.size())); - const int64_t null_count = *rows_read - non_null_count; - if (leaf_batch.values_written() != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet record read result for column {}: values={}, " - "records={}, nulls={}", - _name, leaf_batch.values_written(), *rows_read, null_count); - } - } else if (!is_binary_value && !leaf_batch.read_dense_for_nullable() && - leaf_batch.values_written() != *rows_read) { - return Status::Corruption( - "Invalid parquet record read result for column {}: values={}, records={}", _name, - leaf_batch.values_written(), *rows_read); - } - - RETURN_IF_ERROR(reader.append_values(leaf_batch, *rows_read, &null_map, column)); - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::skip_records(int64_t rows) { - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - if (rows <= 0) { - return Status::OK(); - } - int64_t skipped_rows = 0; - try { - _record_reader->Reset(); - while (skipped_rows < rows) { - const int64_t skipped = _record_reader->SkipRecords(rows - skipped_rows); - if (skipped <= 0) { - return Status::Corruption( - "Failed to skip parquet records for column {}: skipped {} of {} rows", - _name, skipped_rows, rows); - } - skipped_rows += skipped; - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -int64_t ScalarColumnReader::page_filtered_rows_to_skip(int64_t rows) const { - if (_page_skip_plan == nullptr || rows <= 0) { - return 0; - } - const int64_t skip_end = _row_group_rows_read + rows; - int64_t filtered_rows = 0; - for (const auto& range : _page_skip_plan->skipped_ranges) { - const int64_t range_end = range.start + range.length; - if (range_end <= _row_group_rows_read) { - continue; - } - if (range.start >= skip_end) { - break; - } - const int64_t start = std::max(range.start, _row_group_rows_read); - const int64_t end = std::min(range_end, skip_end); - if (start < end) { - // Scheduler gap skips are derived from page-index selected_ranges. A page-filtered - // range can only overlap such a gap when the whole data page is outside every selected - // range, so partial overlap would mean the planner and scheduler are out of sync. - DORIS_CHECK(start == range.start); - DORIS_CHECK(end == range_end); - filtered_rows += end - start; - } - } - return filtered_rows; -} - -void ScalarColumnReader::advance_rows_read(int64_t rows) { - DORIS_CHECK(rows >= 0); - _row_group_rows_read += rows; -} - -Status ScalarColumnReader::skip(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - const int64_t page_filtered_rows = page_filtered_rows_to_skip(rows); - DORIS_CHECK(page_filtered_rows <= rows); - const int64_t record_reader_skip_rows = rows - page_filtered_rows; - RETURN_IF_ERROR(skip_records(record_reader_skip_rows)); - advance_rows_read(rows); - return Status::OK(); -} - -Status ScalarColumnReader::select_with_dictionary_filter(const SelectionVector& sel, - uint16_t selected_rows, int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, - IColumn::Filter* row_filter, - bool* used_filter) { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(used_filter != nullptr); - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); - *used_filter = false; - row_filter->clear(); - row_filter->reserve(selected_rows); - - const auto ranges = selection_to_ranges(sel, selected_rows); - int64_t cursor = 0; - for (const auto& range : ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument( - "Invalid parquet dictionary selection range [{}, {}) for column {}", - range.start, range.start + range.length, _name); - } - RETURN_IF_ERROR(skip(range.start - cursor)); - - int64_t range_rows_read = 0; - RETURN_IF_ERROR(read_range_with_dictionary_filter(range.length, dictionary_filter, column, - row_filter, &range_rows_read, - used_filter)); - if (!*used_filter) { - return Status::OK(); - } - if (range_rows_read != range.length) { - return Status::Corruption( - "Parquet dictionary selected read returned {} rows, expected {} rows for " - "column {}", - range_rows_read, range.length, _name); - } - cursor = range.start + range.length; - } - RETURN_IF_ERROR(skip(batch_rows - cursor)); - if (_profile.reader_select_rows != nullptr) { - COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); - } - return Status::OK(); -} - -Status ScalarColumnReader::read_range_with_dictionary_filter( - int64_t rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* rows_read, bool* used_filter) { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(rows_read != nullptr); - DORIS_CHECK(used_filter != nullptr); - DORIS_CHECK(_record_reader != nullptr); - if (!_record_reader->read_dictionary()) { - *used_filter = false; - return Status::OK(); - } - - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(leaf_reader().read_batch(rows, &leaf_batch, rows_read)); - int64_t matched_rows = 0; - RETURN_IF_ERROR(append_dictionary_filtered_values(leaf_batch.binary_chunks(), dictionary_filter, - column, row_filter, &matched_rows, - used_filter)); - if (!*used_filter) { - return Status::Corruption( - "Parquet dictionary reader did not return dictionary batches for column {}", _name); - } - if (row_filter->size() < static_cast(*rows_read)) { - return Status::Corruption( - "Parquet dictionary filter produced too few row decisions for column {}: " - "filter={}, rows={}", - _name, row_filter->size(), *rows_read); - } - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(matched_rows != nullptr); - DORIS_CHECK(used_filter != nullptr); - *matched_rows = 0; - *used_filter = false; - - std::vector selected_values; - for (const auto& chunk : chunks) { - DORIS_CHECK(chunk != nullptr); - const auto* dict_array = dynamic_cast(chunk.get()); - if (dict_array == nullptr) { - // The caller has already consumed rows from a DictionaryRecordReader. Falling back to a - // normal selected read would desynchronize the Parquet stream, so absence of a - // DictionaryArray is reported as corruption by read_range_with_dictionary_filter(). - return Status::OK(); - } - *used_filter = true; - const auto& dictionary = dict_array->dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - _name); - } - - // Dictionary predicates are evaluated once against the dictionary page and produce a - // dictionary-entry bitmap. DATA_PAGE rows then only need an integer-index lookup. NULL rows - // do not have a dictionary entry and cannot satisfy the supported equality/IN predicates. - for (int64_t row = 0; row < dict_array->length(); ++row) { - bool keep = false; - if (!dict_array->IsNull(row)) { - const int64_t dictionary_index = dict_array->GetValueIndex(row); - if (dictionary_index >= 0 && - dictionary_index < static_cast(dictionary_filter.size())) { - keep = dictionary_filter[static_cast(dictionary_index)] != 0; - } - if (keep) { - RETURN_IF_ERROR(append_arrow_binary_dictionary_value( - _name, *dictionary, dictionary_index, &selected_values)); - ++*matched_rows; - } - } - row_filter->push_back(keep ? 1 : 0); - } - } - - if (!*used_filter) { - return Status::OK(); - } - return append_decoded_binary_values(selected_values, column); -} - -Status ScalarColumnReader::append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const { - DecodedColumnView view; - view.value_kind = decoded_value_kind(_type_descriptor); - view.row_count = static_cast(values.size()); - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.fixed_length = _type_descriptor.fixed_length; - view.binary_values = &values; - - SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(null_map.data() + old_null_map_size, 0, null_map.size() - old_null_map_size); - return Status::OK(); - } - return _type->get_serde()->read_column_from_decoded_values(*column, view); - } - - NullMap null_map(values.size(), 0); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - return _type->get_serde()->read_column_from_decoded_values(*column, view); -} - -// The value index stream must advance on those null slots, otherwise later payload values shift. -Status ScalarColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - RETURN_IF_ERROR(leaf_reader().read_nested_batch(rows, materialized_slot_definition_level, - _nested_batch.get(), _repetition_level)); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(leaf_reader().read_nested_levels_batch(rows, _nested_batch.get())); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar build result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - // The levels-only loader intentionally does not populate value-slot metadata or payload - // buffers. Derive the logical slot threshold from the schema, exactly as load_nested_batch() - // does, so this consumer works for both loaded batch forms. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_read = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_read < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - if (def_level == _definition_level) { - RETURN_IF_ERROR(append_scalar_batch_value(*this, *_nested_batch, current_level_idx, - &value_cursor, column)); - } else { - if (!_type->is_nullable() && def_level >= _nullable_definition_level) { - return Status::Corruption( - "Parquet scalar column {} contains null for non-nullable field", _name); - } - column->insert_default(); - } - ++*values_read; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - if (values_consumed == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar consume result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - // A levels-only batch intentionally has no value-slot metadata. Reconstruct the same logical - // slot threshold used by load_nested_batch(): a nullable leaf owns a slot at one definition - // level below a non-null value, while a required leaf owns a slot only at its full definition - // level. For example, an empty ARRAY boundary must not be consumed as a STRING value. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_consumed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_consumed < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - RETURN_IF_ERROR(validate_nested_value(current_level_idx, false)); - ++*values_consumed; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::append_nested_value(int64_t level_idx, MutableColumnPtr& column) const { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar append result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return append_scalar_batch_value(*this, *_nested_batch, level_idx, &value_cursor, column); - } - if (!_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable value", - _name); - } - column->insert_default(); - return Status::OK(); -} - -Status ScalarColumnReader::validate_nested_value(int64_t level_idx, bool require_non_null) const { - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return Status::OK(); - } - if (require_non_null || !_type->is_nullable()) { - return Status::Corruption("Parquet scalar column {} contains null for non-nullable field", - _name); - } - return Status::OK(); -} - -const std::vector& ScalarColumnReader::nested_definition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->def_levels; -} - -const std::vector& ScalarColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->rep_levels; -} - -int64_t ScalarColumnReader::nested_levels_written() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->levels_written; -} - -bool ScalarColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.h b/be/src/format_v2/parquet/reader/scalar_column_reader.h deleted file mode 100644 index 5342baa803eca0..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.h +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "core/string_ref.h" -#include "format_v2/parquet/parquet_type.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris::format::parquet { - -struct ScalarColumnReaderTestAccess; - -// load_nested_batch() / build_nested_column() -class ScalarColumnReader final : public ParquetColumnReader { - friend class MapColumnReader; - friend struct ScalarColumnReaderTestAccess; - -public: - ScalarColumnReader(const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan = nullptr, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - ParquetColumnReaderProfile profile = {}); - ~ScalarColumnReader() override; - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - bool* used_filter) override; - - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - -private: - Status append_nested_value(int64_t level_idx, MutableColumnPtr& column) const; - Status validate_nested_value(int64_t level_idx, bool require_non_null) const; - Status read_range_with_dictionary_filter(int64_t rows, const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - int64_t* rows_read, bool* used_filter); - Status append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const; - Status append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const; - - const ::parquet::ColumnDescriptor* descriptor() const { return _descriptor; } - - ParquetLeafReader leaf_reader() const { - return ParquetLeafReader(_descriptor, _type_descriptor, _type, _name, _record_reader, - _profile, _timezone, _enable_strict_mode); - } - - void advance_rows_read(int64_t rows); - Status skip_records(int64_t rows); - int64_t page_filtered_rows_to_skip(int64_t rows) const; - - const ::parquet::ColumnDescriptor* _descriptor = nullptr; // Arrow column descriptor - ParquetTypeDescriptor _type_descriptor; // type encoding information - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader - const ParquetPageSkipPlan* _page_skip_plan = - nullptr; // page-index pruning result (may be nullptr) - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - int64_t _row_group_rows_read = 0; // rows read in the current row group (cursor) - std::unique_ptr _nested_batch; // intermediate result for nested reads -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.cpp b/be/src/format_v2/parquet/reader/struct_column_reader.cpp deleted file mode 100644 index 5abe7abe75e9a2..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.cpp +++ /dev/null @@ -1,287 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/struct_column_reader.h" - -#include -#include -#include -#include - -#include "core/column/column_struct.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { - -ParquetColumnReader* StructColumnReader::shape_source_reader() const { - for (const auto& child : _children) { - auto* child_reader = child.get(); - DORIS_CHECK(child_reader != nullptr); - if (!child_reader->is_or_has_repeated_child()) { - return child_reader; - } - } - if (_children.empty()) { - return nullptr; - } - return _children[0].get(); -} - -Status StructColumnReader::advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const { - DORIS_CHECK(child_reader != nullptr); - const int64_t next_child_cursor = parent_level_idx + 1; - if (auto* scalar_child = dynamic_cast(child_reader)) { - if (next_child_cursor > scalar_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - scalar_child->name(), _name); - } - scalar_child->set_nested_build_level_cursor( - std::max(scalar_child->nested_build_level_cursor(), next_child_cursor)); - return Status::OK(); - } - if (auto* struct_child = dynamic_cast(child_reader); - struct_child != nullptr && !struct_child->is_or_has_repeated_child()) { - if (next_child_cursor > struct_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - struct_child->name(), _name); - } - struct_child->set_nested_build_level_cursor( - std::max(struct_child->nested_build_level_cursor(), next_child_cursor)); - for (auto& grandchild : struct_child->_children) { - RETURN_IF_ERROR(struct_child->advance_child_past_null_parent(grandchild.get(), - parent_level_idx)); - } - return Status::OK(); - } - - int64_t child_cursor = child_reader->nested_build_level_cursor(); - const auto& child_rep_levels = child_reader->nested_repetition_levels(); - const int64_t child_levels_written = child_reader->nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!child_reader->is_or_has_repeated_child() || child_rep_level <= _repetition_level) { - break; - } - } - child_reader->set_nested_build_level_cursor(child_cursor); - return Status::OK(); -} - -Status StructColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status StructColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status StructColumnReader::load_nested_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::load_nested_levels_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_levels_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet struct build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status StructColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument( - "Invalid parquet struct process result pointer for column {}", _name); - } - if (_children.empty()) { - if (column != nullptr) { - (*column)->resize((*column)->size() + static_cast(length_upper_bound)); - } - *values_processed = length_upper_bound; - return Status::OK(); - } - ColumnStruct* struct_column = nullptr; - NullMap* parent_null_map = nullptr; - if (column != nullptr) { - struct_column = struct_column_from_output(*column); - DORIS_CHECK(struct_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - } - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - const auto& def_levels = shape_reader->nested_definition_levels(); - const auto& rep_levels = shape_reader->nested_repetition_levels(); - const int64_t levels_written = shape_reader->nested_levels_written(); - - NullMap parent_nulls; - std::vector parent_level_indices; - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < levels_written) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = - !shape_reader->is_or_has_repeated_child() || rep_level <= _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (def_level < _repeated_ancestor_definition_level) { - continue; - } - if (shape_reader->is_or_has_repeated_child() && rep_level > _repetition_level) { - continue; - } - const bool parent_is_null = def_level < _nullable_definition_level; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption( - "Parquet STRUCT column {} contains null for non-nullable struct", _name); - } - parent_nulls.push_back(parent_is_null); - parent_level_indices.push_back(current_level_idx); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - std::vector child_columns; - if (column != nullptr) { - child_columns.reserve(struct_column->get_columns().size()); - for (size_t child_idx = 0; child_idx < struct_column->get_columns().size(); ++child_idx) { - child_columns.push_back(struct_column->get_column_ptr(child_idx)->assert_mutable()); - } - } - for (size_t child_idx = 0; child_idx < _children.size(); ++child_idx) { - const int output_idx = _child_output_indices[child_idx]; - if (column != nullptr && output_idx < 0) { - continue; - } - // STRUCT owns row alignment. Child readers consume only present parent rows from their - // level streams; null STRUCT parents become default placeholders in every child column. - // This mirrors Arrow's separation between struct validity and child array materialization, - // and avoids asking scalar/list/map children to invent values for an absent parent. - int64_t pending_present_rows = 0; - int64_t total_child_rows = 0; - auto flush_present_rows = [&]() -> Status { - if (pending_present_rows == 0) { - return Status::OK(); - } - int64_t child_rows = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_children[child_idx]->build_nested_column( - pending_present_rows, child_columns[output_idx], &child_rows)); - } else { - RETURN_IF_ERROR(_children[child_idx]->consume_nested_column(pending_present_rows, - &child_rows)); - } - if (child_rows != pending_present_rows) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), child_rows, pending_present_rows, _name); - } - total_child_rows += child_rows; - pending_present_rows = 0; - return Status::OK(); - }; - for (size_t parent_idx = 0; parent_idx < parent_nulls.size(); ++parent_idx) { - const auto parent_is_null = parent_nulls[parent_idx]; - if (!parent_is_null) { - ++pending_present_rows; - continue; - } - RETURN_IF_ERROR(flush_present_rows()); - if (column != nullptr) { - child_columns[output_idx]->insert_default(); - } - RETURN_IF_ERROR(advance_child_past_null_parent(_children[child_idx].get(), - parent_level_indices[parent_idx])); - ++total_child_rows; - } - RETURN_IF_ERROR(flush_present_rows()); - if (total_child_rows != *values_processed) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), total_child_rows, *values_processed, _name); - } - } - if (column != nullptr) { - for (size_t child_idx = 0; child_idx < child_columns.size(); ++child_idx) { - struct_column->get_column_ptr(child_idx) = std::move(child_columns[child_idx]); - } - append_parent_nulls(parent_null_map, parent_nulls); - } - return Status::OK(); -} - -const std::vector& StructColumnReader::nested_definition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_definition_levels(); -} - -const std::vector& StructColumnReader::nested_repetition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_repetition_levels(); -} - -int64_t StructColumnReader::nested_levels_written() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_levels_written(); -} - -bool StructColumnReader::is_or_has_repeated_child() const { - auto* shape_reader = shape_source_reader(); - return shape_reader != nullptr && shape_reader->is_or_has_repeated_child(); -} - -void StructColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - for (auto& child : _children) { - DORIS_CHECK(child != nullptr); - child->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - } -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.h b/be/src/format_v2/parquet/reader/struct_column_reader.h deleted file mode 100644 index 3c2d6904cb36f4..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.h +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class StructColumnReader final : public ParquetColumnReader { -public: - StructColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::vector> children, - std::vector child_output_indices, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _children(std::move(children)), - _child_output_indices(std::move(child_output_indices)) { - DCHECK_EQ(_children.size(), _child_output_indices.size()); - } - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - ParquetColumnReader* shape_source_reader() const; - Status advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const; - - std::vector> _children; // projected child readers - std::vector _child_output_indices; // child reader -> struct output position mapping -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h index 589154d4acc0e4..ab2bf93c785edc 100644 --- a/be/src/format_v2/parquet/selection_vector.h +++ b/be/src/format_v2/parquet/selection_vector.h @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -33,11 +34,11 @@ struct RowRange { struct ParquetPageSkipPlan { int leaf_column_id = -1; // Page ordinal is the data-page ordinal in the column chunk. It intentionally excludes - // dictionary pages, matching Arrow PageReader::set_data_page_filter(). + // dictionary pages, matching the native page reader's ordinal domain. std::vector skipped_pages; std::vector skipped_page_compressed_sizes; - // Row ranges covered by skipped data pages. ScalarColumnReader uses these ranges to avoid - // calling RecordReader::SkipRecords() again for pages already skipped by Arrow. + // Row ranges covered by skipped data pages. NativeColumnReader uses these ranges to avoid + // consuming logical rows twice after the page reader has already skipped their payload. std::vector skipped_ranges; bool empty() const { return skipped_ranges.empty(); } @@ -66,6 +67,7 @@ class SelectionVector { _owned.clear(); _data = data; _size = count; + ++_generation; } void resize(size_t count) { @@ -75,12 +77,14 @@ class SelectionVector { for (size_t idx = 0; idx < count; ++idx) { _data[idx] = static_cast(idx); } + ++_generation; } void clear() { _owned.clear(); _data = nullptr; _size = 0; + ++_generation; } size_t size() const { return _size; } @@ -98,7 +102,29 @@ class SelectionVector { return _data[idx]; } - void set_index(size_t idx, Index value) { _data[idx] = value; } + void set_index(size_t idx, Index value) { + _data[idx] = value; + ++_generation; + } + + Status materialize_filter(size_t count, int64_t batch_rows, const uint8_t** filter) const { + DORIS_CHECK(filter != nullptr); + RETURN_IF_ERROR(verify(count, batch_rows)); + if (_filter_generation != _generation || _filter_count != count || + _filter_batch_rows != batch_rows) { + // Selection is shared by all readers in one scheduler batch. Cache its dense bitmap so + // a wide lazy projection does not rebuild the same O(batch_rows) filter per column. + _filter.assign(static_cast(batch_rows), 0); + for (size_t idx = 0; idx < count; ++idx) { + _filter[get_index(idx)] = 1; + } + _filter_generation = _generation; + _filter_count = count; + _filter_batch_rows = batch_rows; + } + *filter = _filter.data(); + return Status::OK(); + } Status verify(size_t count, int64_t batch_rows) const { if (batch_rows < 0) { @@ -135,13 +161,19 @@ class SelectionVector { std::vector _owned; Index* _data = nullptr; size_t _size = 0; + uint64_t _generation = 0; + mutable std::vector _filter; + mutable uint64_t _filter_generation = std::numeric_limits::max(); + mutable size_t _filter_count = 0; + mutable int64_t _filter_batch_rows = -1; }; -inline std::vector selection_to_ranges(const SelectionVector& selection, - uint16_t selected_rows) { - std::vector ranges; +inline void selection_to_ranges(const SelectionVector& selection, uint16_t selected_rows, + std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); if (selected_rows == 0) { - return ranges; + return; } int64_t range_start = selection.get_index(0); @@ -152,11 +184,17 @@ inline std::vector selection_to_ranges(const SelectionVector& selectio previous = current; continue; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); range_start = current; previous = current; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); +} + +inline std::vector selection_to_ranges(const SelectionVector& selection, + uint16_t selected_rows) { + std::vector ranges; + selection_to_ranges(selection, selected_rows, &ranges); return ranges; } diff --git a/be/src/format_v2/table/hive_reader.cpp b/be/src/format_v2/table/hive_reader.cpp index 74a95fd43965cf..619e9959a1bfa4 100644 --- a/be/src/format_v2/table/hive_reader.cpp +++ b/be/src/format_v2/table/hive_reader.cpp @@ -102,11 +102,16 @@ void add_name_mapping(std::vector* name_mapping, const std::string& } // namespace Status HiveReader::prepare_split(const format::SplitReadOptions& options) { - if (options.current_split_format != _format) { - return Status::InternalError( - "Hive scan expects all splits to use the same file format, " - "initialized_format={}, current_split_format={}", - static_cast(_format), static_cast(options.current_split_format)); + { + // Keep derived validation visible without overlapping the base scopes on the same timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + if (options.current_split_format != _format) { + return Status::InternalError( + "Hive scan expects all splits to use the same file format, " + "initialized_format={}, current_split_format={}", + static_cast(_format), static_cast(options.current_split_format)); + } } return format::TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 4294d51d043d9e..271f14a7b483eb 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -28,13 +28,20 @@ namespace doris::format::hudi { Status HudiReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.__isset.hudi_params && - options.current_range.table_format_params.hudi_params.__isset.schema_id) { - _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.hudi_params && + options.current_range.table_format_params.hudi_params.__isset.schema_id) { + _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -66,6 +73,8 @@ Status HudiHybridReader::init(format::TableReadOptions&& options) { } Status HudiHybridReader::prepare_split(const format::SplitReadOptions& options) { + // A newly selected child initializes against the same scanner profile. Keep hybrid dispatch + // outside those shared counters so first-split initialization is counted exactly once. RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); @@ -81,6 +90,11 @@ bool HudiHybridReader::current_split_pruned() const { return _current_split_reader->current_split_pruned(); } +bool HudiHybridReader::current_split_uses_metadata_count() const { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->current_split_uses_metadata_count(); +} + Status HudiHybridReader::abort_split() { DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->abort_split(); @@ -111,11 +125,47 @@ void HudiHybridReader::set_batch_size(size_t batch_size) { } } +Status HudiHybridReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + // The wrapper snapshot initializes future children, while every existing child needs the same + // late RF immediately so active and later reused splits keep identical predicate ownership. + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + RETURN_IF_ERROR(format::TableReader::append_conjuncts(conjuncts)); + if (_native_reader != nullptr) { + RETURN_IF_ERROR(_native_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + if (_jni_reader != nullptr) { + RETURN_IF_ERROR(_jni_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + return Status::OK(); +} + +const format::MaterializedBlockStats& HudiHybridReader::last_materialized_block_stats() const { + // FileScannerV2 budgets cooperative work from the child that actually materialized the block. + return _current_split_reader != nullptr ? _current_split_reader->last_materialized_block_stats() + : format::TableReader::last_materialized_block_stats(); +} + +int64_t HudiHybridReader::condition_cache_hit_count() const { + // Keep the wrapper count cumulative across native/JNI dispatch so scanner-level delta + // accounting neither loses a child hit nor observes a counter reset on a split switch. + return (_native_reader == nullptr ? 0 : _native_reader->condition_cache_hit_count()) + + (_jni_reader == nullptr ? 0 : _jni_reader->condition_cache_hit_count()); +} + Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { DORIS_CHECK(_scan_params != nullptr); if (_is_jni_split(*_scan_params, options.current_range)) { if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -123,7 +173,15 @@ Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOpt format::FileFormat file_format; RETURN_IF_ERROR(_to_file_format(*_scan_params, options.current_range, &file_format)); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::hudi::HudiReader::create_unique(); + } +#else _native_reader = format::hudi::HudiReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); @@ -139,12 +197,14 @@ Status HudiHybridReader::_init_child_reader(format::TableReader* reader, RETURN_IF_ERROR(reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = _scan_params, .io_ctx = _io_ctx, .runtime_state = _runtime_state, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, + .push_down_count_columns = _push_down_count_columns, .condition_cache_digest = _condition_cache_digest, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index e22c6bd866f061..c13ac1215d72ee 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -60,9 +61,13 @@ class HudiHybridReader final : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; + bool current_split_uses_metadata_count() const override; Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; + const format::MaterializedBlockStats& last_materialized_block_stats() const override; + int64_t condition_cache_hit_count() const override; #ifdef BE_TEST void TEST_install_batch_size_children() { @@ -72,6 +77,16 @@ class HudiHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_condition_cache_hits(int64_t native_hits, int64_t jni_hits) { + _native_reader->TEST_set_condition_cache_hit_count(native_hits); + _jni_reader->TEST_set_condition_cache_hit_count(jni_hits); + } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -87,6 +102,10 @@ class HudiHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle native parquet/orc splits std::unique_ptr _jni_reader; // handle MOR JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::hudi diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index ef13f585fb6a6d..6ab67caa77da03 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -35,6 +35,7 @@ #include "core/types.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format/table/parquet_utils.h" +#include "format_v2/table/iceberg_schema_utils.h" #include "runtime/descriptors.h" #include "runtime/runtime_state.h" @@ -132,24 +133,25 @@ void set_iceberg_delete_field_id(ColumnDefinition* column) { } } -bool has_field_id(const std::vector& schema) { - for (const auto& field : schema) { - if (!field.has_identifier_field_id()) { - return false; - } - if (!has_field_id(field.children)) { - return false; - } - } - return true; -} - class PositionDeleteFileTableReader final : public format::TableReader { protected: format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && has_field_id(_data_reader.file_schema) - ? format::TableColumnMappingMode::BY_FIELD_ID - : format::TableColumnMappingMode::BY_NAME; + const bool has_field_ids = supports_iceberg_scan_semantics_v1(_scan_params) + ? schema_has_any_field_id(_data_reader.file_schema) + : schema_has_all_field_ids(_data_reader.file_schema); + if (!_data_reader.file_schema.empty() && has_field_ids) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; + } + + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + options->enable_row_lineage_virtual_columns = true; + // Parquet may preserve a selected complex wrapper without its own ID; position-delete row + // projection must use the same descendant-ID fallback as ordinary Iceberg data scans. + options->allow_idless_complex_wrapper_projection = + supports_iceberg_scan_semantics_v1(_scan_params) && + _format == format::FileFormat::PARQUET; } }; @@ -161,13 +163,25 @@ Status IcebergPositionDeleteSysTableV2Reader::prepare_split( const format::SplitReadOptions& options) { RETURN_IF_ERROR(close()); RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + if (current_split_pruned()) { + return Status::OK(); + } + // This synthetic reader has no physical schema where a predicate can be localized, so every + // split predicate must run after its system-table columns have been materialized. + RETURN_IF_ERROR(_prepare_all_conjuncts_as_remaining()); + // The inner delete-file reader has distinct counters, so the outer preparation can safely + // contain its cache miss/open work without re-entering the same RuntimeProfile timer. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); _current_range = options.current_range; _has_split = true; return _init_split(); } Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); + _reset_materialized_block_stats(); DORIS_CHECK(block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(block->columns() == _projected_columns.size()); @@ -185,9 +199,19 @@ Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) return Status::OK(); } - size_t read_rows = 0; if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { - return _append_deletion_vector_block(block, &read_rows, eos); + size_t read_rows = 0; + RETURN_IF_ERROR(_append_deletion_vector_block(block, &read_rows, eos)); + if (read_rows > 0) { + _record_materialized_block_stats(*block, read_rows); + RETURN_IF_ERROR(_filter_remaining_conjuncts(block, &read_rows)); + } + if (read_rows == 0) { + // Yield after one deletion-vector batch so cancellation and Scanner row budgets are + // observed even when residual predicates reject every synthesized row. + block->clear_column_data(_projected_columns.size()); + } + return Status::OK(); } DORIS_CHECK(_position_reader != nullptr); @@ -201,8 +225,18 @@ Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) RETURN_IF_ERROR(_position_reader->get_block(&delete_block, &position_reader_eof)); const size_t delete_rows = delete_block.rows(); if (delete_rows > 0) { + size_t read_rows = 0; RETURN_IF_ERROR( _append_position_delete_block(block, delete_block, delete_rows, &read_rows)); + _record_materialized_block_stats(*block, read_rows); + RETURN_IF_ERROR(_filter_remaining_conjuncts(block, &read_rows)); + if (read_rows == 0) { + // A filtered materialized batch is still progress; return it to Scanner instead of + // consuming an unbounded number of position-delete batches in this call. + block->clear_column_data(_projected_columns.size()); + *eos = false; + return Status::OK(); + } *eos = false; return Status::OK(); } @@ -307,6 +341,15 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { std::vector projected_columns; RETURN_IF_ERROR(_build_delete_file_projected_columns(&projected_columns)); + static constexpr const char* kPositionReaderProfile = "IcebergPositionDeleteFileReader"; + if (_position_reader_profile == nullptr) { + _position_reader_profile = _scanner_profile->get_child(kPositionReaderProfile); + if (_position_reader_profile == nullptr) { + // The outer system-table reader calls the inner reader synchronously. Giving both the + // same profile would nest identical counter pointers and double-count every timer. + _position_reader_profile = _scanner_profile->create_child(kPositionReaderProfile); + } + } _position_reader = std::make_unique(); RETURN_IF_ERROR(_position_reader->init({ .projected_columns = std::move(projected_columns), @@ -315,7 +358,7 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { .scan_params = _scan_params, .io_ctx = _io_ctx, .runtime_state = _runtime_state, - .scanner_profile = _scanner_profile, + .scanner_profile = _position_reader_profile, .file_slot_descs = nullptr, .push_down_agg_type = TPushAggOp::type::NONE, .condition_cache_digest = 0, diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h index 9c802e726053cb..09756daaabe4da 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h @@ -76,6 +76,7 @@ class IcebergPositionDeleteSysTableV2Reader final : public format::TableReader { const TIcebergDeleteFileDesc* _delete_file_desc = nullptr; DeleteFileKind _delete_file_kind = DeleteFileKind::POSITION_DELETE; std::unique_ptr _position_reader; + RuntimeProfile* _position_reader_profile = nullptr; std::vector _read_columns; ColumnPtr _partition_value; roaring::Roaring64Map _dv_positions; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index b817a6d1eadbf5..407d5924e9f64a 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -44,6 +44,7 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "io/file_factory.h" +#include "util/debug_points.h" #include "util/url_coding.h" namespace doris::format::iceberg { @@ -91,7 +92,7 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit return Status::OK(); } - Field initial_default; + VExprSPtr literal; if (table_field.initial_default_value_is_base64 || table_field.type->get_primitive_type() == TYPE_VARBINARY) { // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its @@ -104,19 +105,26 @@ static Status build_missing_equality_delete_key_expr(const format::ColumnDefinit table_field.name); } if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { - initial_default = Field::create_field(StringView(decoded_default)); + const auto initial_default = + Field::create_field(StringView(decoded_default)); + // VLiteral must copy the borrowed StringView while decoded_default is alive; UUID and + // long FIXED defaults otherwise retain a pointer into freed decode storage. + literal = VLiteral::create_shared(table_field.type, initial_default); } else { DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); - initial_default = Field::create_field(decoded_default); + literal = VLiteral::create_shared(table_field.type, + Field::create_field(decoded_default)); } } else { // An added field's initial default is its logical value in every older data file that lacks // the physical column. FE normalizes the string for the current Doris table type. + Field initial_default; RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( *table_field.initial_default_value, initial_default)); + literal = VLiteral::create_shared(table_field.type, initial_default); } - auto literal = VLiteral::create_shared(table_field.type, initial_default); + DORIS_CHECK(literal != nullptr); if (table_field.type->equals(*delete_key_type)) { *key_expr = std::move(literal); return Status::OK(); @@ -225,25 +233,31 @@ Status IcebergTableReader::PositionDeleteRowsCollector::collect(const Block& blo } Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options) { - _row_lineage_columns = {}; - _iceberg_params.reset(); - _delete_predicates_initialized = false; - _position_delete_rows_storage.clear(); - _equality_delete_filters.clear(); - _split_cache = options.cache; - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.__isset.iceberg_params) { - const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; - _iceberg_params = iceberg_params; - if (iceberg_params.__isset.first_row_id) { - _row_lineage_columns.first_row_id = iceberg_params.first_row_id; - } - if (iceberg_params.__isset.last_updated_sequence_number) { - _row_lineage_columns.last_updated_sequence_number = - iceberg_params.last_updated_sequence_number; + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _row_lineage_columns = {}; + _iceberg_params.reset(); + _delete_predicates_initialized = false; + _position_delete_rows_storage.clear(); + _equality_delete_filters.clear(); + _split_cache = options.cache; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.iceberg_params) { + const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; + _iceberg_params = iceberg_params; + if (iceberg_params.__isset.first_row_id) { + _row_lineage_columns.first_row_id = iceberg_params.first_row_id; + } + if (iceberg_params.__isset.last_updated_sequence_number) { + _row_lineage_columns.last_updated_sequence_number = + iceberg_params.last_updated_sequence_number; + } } } RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -255,6 +269,8 @@ Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options if (_is_table_level_count_active()) { return Status::OK(); } + DBUG_EXECUTE_IF("IcebergTableReader.prepare_split.before_delete_file_scan", + DBUG_RUN_CALLBACK()); RETURN_IF_ERROR(_init_delete_predicates(options.current_range.table_format_params)); return Status::OK(); } @@ -387,10 +403,8 @@ Status IcebergTableReader::_parse_deletion_vector_file(const TTableFormatFileDes if (deletion_vector == nullptr) { return Status::OK(); } - if (!deletion_vector->__isset.content_offset || - !deletion_vector->__isset.content_size_in_bytes) { - return Status::InternalError("Deletion vector is missing content offset or length"); - } + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_iceberg_deletion_vector_descriptor(*deletion_vector, bytes_read)); const std::string data_file_path = iceberg_params.__isset.original_file_path ? iceberg_params.original_file_path @@ -398,7 +412,7 @@ Status IcebergTableReader::_parse_deletion_vector_file(const TTableFormatFileDes desc->key = build_iceberg_deletion_vector_cache_key(data_file_path, *deletion_vector); desc->path = deletion_vector->path; desc->start_offset = deletion_vector->content_offset; - desc->size = deletion_vector->content_size_in_bytes; + desc->size = static_cast(bytes_read); desc->file_size = -1; desc->format = DeleteFileDesc::Format::ICEBERG; *has_delete_file = true; @@ -568,14 +582,13 @@ void IcebergTableReader::_append_equality_delete_row_count_carrier( DORIS_CHECK(request != nullptr); // Columnar readers establish a filter batch's row count from predicate columns. If all // equality keys are missing, the predicate consists only of NULL literals and the filter block - // would otherwise have zero rows. Read one physical column eagerly as a row-count carrier; - // normal final materialization ignores this hidden dependency. - const auto carrier_it = std::ranges::find_if( - _data_reader.file_schema, [](const format::ColumnDefinition& field) { - return field.column_type == format::ColumnType::DATA_COLUMN; - }); - DORIS_CHECK(carrier_it != _data_reader.file_schema.end()); - _append_file_scan_column(request, format::LocalColumnId(carrier_it->file_local_id()), + // would otherwise have zero rows. Use the virtual row-position column as the carrier instead + // of an arbitrary physical column. For example, a data file may start with an unsupported + // TIME_MILLIS leaf while the query projects only a supported `id`; selecting that TIME leaf as + // a hidden carrier would make Parquet reject a column the query never requested. Row position + // has one value per input row in both Parquet and ORC, is already used by delete predicates, + // and is explicitly excluded from physical logical-type validation. + _append_file_scan_column(request, format::LocalColumnId(format::ROW_POSITION_COLUMN_ID), &request->predicate_columns); } @@ -645,10 +658,13 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && scan_params.enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = + scan_params.__isset.enable_mapping_varbinary && scan_params.enable_mapping_varbinary; if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { + // Delete and data files must parse raw binary fields with the same scan-level mapping. *reader = std::make_unique( system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, - enable_mapping_timestamp_tz); + enable_mapping_timestamp_tz, enable_mapping_varbinary); } else { *reader = std::make_unique(system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, @@ -832,25 +848,22 @@ Status IcebergTableReader::_load_equality_delete_file(const TIcebergDeleteFileDe RETURN_IF_ERROR(_resolve_equality_delete_fields(delete_file, schema, &delete_fields, result)); auto request = std::make_shared(); - auto build_block = [](const std::vector& fields) -> Block { - Block block; - for (const auto& field : fields) { - block.insert({field.type->create_column(), field.type, field.name}); - } - return block; - }; + Block delete_block_template; for (size_t idx = 0; idx < delete_fields.size(); ++idx) { - const auto local_column_id = format::LocalColumnId(delete_fields[idx].file_local_id()); + const auto& delete_field = delete_fields[idx]; + const auto local_column_id = format::LocalColumnId(delete_field.file_local_id()); request->non_predicate_columns.push_back( format::LocalColumnIndex::top_level(local_column_id)); request->local_positions.emplace(local_column_id, format::LocalIndex(idx)); + delete_block_template.insert( + {delete_field.type->create_column(), delete_field.type, delete_field.name}); } RETURN_IF_ERROR(reader->open(request)); - MutableBlock mutable_delete_block(build_block(delete_fields)); + MutableBlock mutable_delete_block(delete_block_template.clone_empty()); bool eof = false; while (!eof) { - Block block = build_block(delete_fields); + Block block = delete_block_template.clone_empty(); size_t read_rows = 0; RETURN_IF_ERROR(reader->get_block(&block, &read_rows, &eof)); if (read_rows > 0) { diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index d328f574f2b22b..d28be3d7f98f0b 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -27,6 +27,7 @@ #include "core/block/block.h" #include "format/table/iceberg_delete_file_reader_helper.h" #include "format_v2/file_reader.h" +#include "format_v2/table/iceberg_schema_utils.h" #include "format_v2/table_reader.h" #include "gen_cpp/PlanNodes_types.h" @@ -57,12 +58,22 @@ class IcebergTableReader : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; std::string debug_string() const override; format::TableColumnMappingMode mapping_mode() const override { - return !_data_reader.file_schema.empty() && _has_field_id(_data_reader.file_schema) - ? format::TableColumnMappingMode::BY_FIELD_ID - : format::TableColumnMappingMode::BY_NAME; + const bool has_field_ids = supports_iceberg_scan_semantics_v1(_scan_params) + ? schema_has_any_field_id(_data_reader.file_schema) + : schema_has_all_field_ids(_data_reader.file_schema); + if (!_data_reader.file_schema.empty() && has_field_ids) { + return format::TableColumnMappingMode::BY_FIELD_ID; + } + return format::TableColumnMappingMode::BY_NAME; } protected: + void configure_mapper_options(format::TableColumnMapperOptions* options) const override { + options->enable_row_lineage_virtual_columns = true; + options->allow_idless_complex_wrapper_projection = + supports_iceberg_scan_semantics_v1(_scan_params) && _format == FileFormat::PARQUET; + } + Status materialize_virtual_columns(Block* table_block) override; Status customize_file_scan_request(format::FileScanRequest* file_request) override; @@ -76,26 +87,6 @@ class IcebergTableReader : public format::TableReader { private: struct EqualityDeleteFilter; - - bool _has_field_id(const std::vector& schema) const { - for (const auto& field : schema) { - // TopN lazy materialization asks the file reader to synthesize GLOBAL_ROWID in the - // first-phase scan. That virtual column is not an Iceberg data field and therefore has - // no Iceberg field id. Do not let it downgrade schema-evolution reads to BY_NAME, - // otherwise old data files whose physical names predate a rename (for example, - // table column `new_new_id` stored as file column `id`) are materialized as defaults. - if (field.column_type != format::ColumnType::DATA_COLUMN) { - continue; - } - if (!field.has_identifier_field_id()) { - return false; - } - if (!_has_field_id(field.children)) { - return false; - } - } - return true; - } static constexpr int MIN_SUPPORT_DELETE_FILES_VERSION = 2; static constexpr int POSITION_DELETE = 1; static constexpr int EQUALITY_DELETE = 2; diff --git a/be/src/format_v2/table/iceberg_schema_utils.h b/be/src/format_v2/table/iceberg_schema_utils.h new file mode 100644 index 00000000000000..516c982194bbe4 --- /dev/null +++ b/be/src/format_v2/table/iceberg_schema_utils.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format/table/iceberg_scan_semantics.h" +#include "format_v2/column_data.h" + +namespace doris::format::iceberg { + +inline bool schema_has_any_field_id(const std::vector& schema) { + for (const auto& field : schema) { + // Iceberg's hasIds contract is existential for the whole file. Ignore synthesized columns + // and retain ID projection when any real field, including a nested field, carries an ID. + if (field.column_type != ColumnType::DATA_COLUMN) { + continue; + } + if (field.has_identifier_field_id() || schema_has_any_field_id(field.children)) { + return true; + } + } + return false; +} + +inline bool schema_has_all_field_ids(const std::vector& schema) { + for (const auto& field : schema) { + if (field.column_type != ColumnType::DATA_COLUMN) { + continue; + } + if (!field.has_identifier_field_id() || !schema_has_all_field_ids(field.children)) { + return false; + } + } + return true; +} + +} // namespace doris::format::iceberg diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 258bbb5c021dc0..d815ef81c9b424 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -33,12 +33,19 @@ namespace doris::format::paimon { Status PaimonReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - const auto& paimon_params = options.current_range.table_format_params.paimon_params; - if (paimon_params.__isset.schema_id) { - _split_schema_id = paimon_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + const auto& paimon_params = options.current_range.table_format_params.paimon_params; + if (paimon_params.__isset.schema_id) { + _split_schema_id = paimon_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -75,11 +82,13 @@ Status PaimonReader::_parse_deletion_vector_file(const TTableFormatFileDesc& t_d return Status::OK(); } const auto& deletion_file = table_desc.deletion_file; + size_t bytes_read = 0; + RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read)); desc->key = build_paimon_deletion_vector_cache_key(deletion_file); desc->path = deletion_file.path; desc->start_offset = deletion_file.offset; - desc->size = deletion_file.length + 4; + desc->size = static_cast(bytes_read); desc->file_size = -1; desc->format = DeleteFileDesc::Format::PAIMON; *has_delete_file = true; @@ -91,6 +100,8 @@ Status PaimonHybridReader::init(format::TableReadOptions&& options) { } Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options) { + // Child initialization uses the scanner profile too; hybrid dispatch must not nest the same + // timer around the first native or JNI child and double-count that initialization. RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); @@ -106,6 +117,11 @@ bool PaimonHybridReader::current_split_pruned() const { return _current_split_reader->current_split_pruned(); } +bool PaimonHybridReader::current_split_uses_metadata_count() const { + DORIS_CHECK(_current_split_reader != nullptr); + return _current_split_reader->current_split_uses_metadata_count(); +} + Status PaimonHybridReader::abort_split() { DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->abort_split(); @@ -136,11 +152,47 @@ void PaimonHybridReader::set_batch_size(size_t batch_size) { } } +Status PaimonHybridReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + // The wrapper snapshot initializes future children, while every existing child needs the same + // late RF immediately so active and later reused splits keep identical predicate ownership. + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + RETURN_IF_ERROR(format::TableReader::append_conjuncts(conjuncts)); + if (_native_reader != nullptr) { + RETURN_IF_ERROR(_native_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + if (_jni_reader != nullptr) { + RETURN_IF_ERROR(_jni_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + return Status::OK(); +} + +const format::MaterializedBlockStats& PaimonHybridReader::last_materialized_block_stats() const { + // FileScannerV2 budgets cooperative work from the child that actually materialized the block. + return _current_split_reader != nullptr ? _current_split_reader->last_materialized_block_stats() + : format::TableReader::last_materialized_block_stats(); +} + +int64_t PaimonHybridReader::condition_cache_hit_count() const { + // Both children survive split switches, so the wrapper must publish their cumulative totals; + // returning only the active child would make FileScannerV2's monotonic delta go backwards. + return (_native_reader == nullptr ? 0 : _native_reader->condition_cache_hit_count()) + + (_jni_reader == nullptr ? 0 : _jni_reader->condition_cache_hit_count()); +} + Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadOptions& options) { if (_is_jni_split(options.current_range)) { DCHECK(options.current_split_format == format::FileFormat::JNI); if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -151,7 +203,15 @@ Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadO DCHECK(file_format == format::FileFormat::PARQUET || file_format == format::FileFormat::ORC); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::paimon::PaimonReader::create_unique(); + } +#else _native_reader = format::paimon::PaimonReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); @@ -167,12 +227,14 @@ Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, RETURN_IF_ERROR(reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = _scan_params, .io_ctx = _io_ctx, .runtime_state = _runtime_state, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, + .push_down_count_columns = _push_down_count_columns, .condition_cache_digest = _condition_cache_digest, })); // Zero means no adaptive prediction has been produced yet. Preserve the child's normal diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index fc94777b6dafb3..b4f076fc6e4878 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include "format_v2/table_reader.h" @@ -66,9 +67,13 @@ class PaimonHybridReader final : public format::TableReader { Status prepare_split(const format::SplitReadOptions& options) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; + bool current_split_uses_metadata_count() const override; Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; + const format::MaterializedBlockStats& last_materialized_block_stats() const override; + int64_t condition_cache_hit_count() const override; #ifdef BE_TEST static bool TEST_is_jni_split(const TFileRangeDesc& range) { return _is_jni_split(range); } @@ -83,6 +88,16 @@ class PaimonHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_condition_cache_hits(int64_t native_hits, int64_t jni_hits) { + _native_reader->TEST_set_condition_cache_hit_count(native_hits); + _jni_reader->TEST_set_condition_cache_hit_count(jni_hits); + } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -95,6 +110,10 @@ class PaimonHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle parquet/orc native splits std::unique_ptr _jni_reader; // handle serialized JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::paimon diff --git a/be/src/format_v2/table/remote_doris_reader.cpp b/be/src/format_v2/table/remote_doris_reader.cpp index c67cece6e05b8c..bd2c08b5f2310f 100644 --- a/be/src/format_v2/table/remote_doris_reader.cpp +++ b/be/src/format_v2/table/remote_doris_reader.cpp @@ -20,8 +20,13 @@ #include #include +#include +#include +#include #include +#include #include +#include #include #include @@ -37,7 +42,10 @@ #include "format/arrow/arrow_utils.h" #include "format_v2/materialized_reader_util.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" +#include "runtime/query_context.h" #include "runtime/runtime_state.h" +#include "runtime/thread_context.h" #include "util/timezone_utils.h" namespace doris::format::remote_doris { @@ -63,7 +71,12 @@ Status validate_remote_doris_range(const TFileRangeDesc& range) { class FlightRemoteDorisStream final : public RemoteDorisStream { public: - explicit FlightRemoteDorisStream(const TFileRangeDesc& range) : _range(range) {} + FlightRemoteDorisStream(const TFileRangeDesc& range, std::shared_ptr io_ctx, + RuntimeState* runtime_state, int timeout_seconds) + : _range(range), + _io_ctx(std::move(io_ctx)), + _runtime_state(runtime_state), + _timeout_seconds(std::max(1, timeout_seconds)) {} Status open() { RETURN_IF_ERROR(validate_remote_doris_range(_range)); @@ -74,14 +87,103 @@ class FlightRemoteDorisStream final : public RemoteDorisStream { arrow::flight::Ticket ticket; RETURN_DORIS_STATUS_IF_ERROR( arrow::flight::Ticket::Deserialize(params.ticket).Value(&ticket)); + struct PendingOpen { + std::mutex mutex; + std::condition_variable cv; + bool done = false; + bool abandoned = false; + arrow::Status status = arrow::Status::OK(); + std::unique_ptr client; + std::unique_ptr stream; + }; + auto pending = std::make_shared(); + std::unique_ptr flight_client; RETURN_DORIS_STATUS_IF_ERROR( - arrow::flight::FlightClient::Connect(location).Value(&_flight_client)); - RETURN_DORIS_STATUS_IF_ERROR(_flight_client->DoGet(ticket).Value(&_stream)); + arrow::flight::FlightClient::Connect(location).Value(&flight_client)); + arrow::flight::FlightCallOptions options; + // A Flight deadline covers streaming reads as well as DoGet setup, so a stalled Next() + // cannot outlive the query execution timeout indefinitely. + options.timeout = std::chrono::seconds(_timeout_seconds); + // Start before DoGet because endpoint setup is itself a blocking RPC covered by the same + // query/scanner cancellation contract as streaming Next(). + _cancellation_watcher = std::jthread( + [this](std::stop_token stop_token) { _watch_cancellation(stop_token); }); + + std::shared_ptr resource_ctx; + if (_runtime_state != nullptr && _runtime_state->get_query_ctx() != nullptr) { + resource_ctx = _runtime_state->get_query_ctx()->resource_ctx(); + } + std::thread do_get_thread([pending, options, ticket, resource_ctx, + client = std::move(flight_client)]() mutable { + const auto do_get = [&] { + std::unique_ptr stream; + auto status = client->DoGet(options, ticket).Value(&stream); + { + std::lock_guard lock(pending->mutex); + if (!pending->abandoned) { + pending->status = std::move(status); + pending->client = std::move(client); + pending->stream = std::move(stream); + } else { + // A detached worker must release its query-owned Flight client + // before leaving the task attachment that accounts for it. + client.reset(); + } + pending->done = true; + } + pending->cv.notify_all(); + }; + if (resource_ctx != nullptr) { + SCOPED_ATTACH_TASK(resource_ctx); + do_get(); + } else { + SCOPED_INIT_THREAD_CONTEXT(); + do_get(); + } + }); + bool cancelled_during_open = false; + { + std::unique_lock lock(pending->mutex); + while (!pending->done && !_is_cancelled()) { + pending->cv.wait_for(lock, std::chrono::milliseconds(25)); + } + if (!pending->done) { + pending->abandoned = true; + cancelled_during_open = true; + } + } + if (cancelled_during_open) { + // Arrow 17 exposes no cancellable handle until DoGet returns. Detaching the bounded RPC + // keeps query/scanner shutdown prompt while the call is still capped by its deadline; + // the shared state owns all Arrow objects until that worker exits. + do_get_thread.detach(); + _stop_cancellation_watcher(); + return Status::Cancelled("Remote Doris Flight open was cancelled"); + } + do_get_thread.join(); + if (!pending->status.ok()) { + _stop_cancellation_watcher(); + RETURN_DORIS_STATUS_IF_ERROR(pending->status); + } + { + std::lock_guard lock(_flight_mutex); + _flight_client = std::move(pending->client); + _stream = std::move(pending->stream); + } + if (_is_cancelled()) { + _cancel_flight_call(); + _stop_cancellation_watcher(); + return Status::Cancelled("Remote Doris Flight open was cancelled"); + } return Status::OK(); } Status next(std::shared_ptr* batch) override { DORIS_CHECK(batch != nullptr); + if (_io_ctx != nullptr && _io_ctx->should_stop) { + _cancel_flight_call(); + return Status::Cancelled("Remote Doris Flight read was cancelled"); + } arrow::flight::FlightStreamChunk chunk; RETURN_DORIS_STATUS_IF_ERROR(_stream->Next().Value(&chunk)); *batch = chunk.data; @@ -89,6 +191,13 @@ class FlightRemoteDorisStream final : public RemoteDorisStream { } Status close() override { + _stop_cancellation_watcher(); + { + std::lock_guard lock(_flight_mutex); + if (_stream != nullptr) { + _stream->Cancel(); + } + } _stream.reset(); if (_flight_client != nullptr) { RETURN_DORIS_STATUS_IF_ERROR(_flight_client->Close()); @@ -98,14 +207,65 @@ class FlightRemoteDorisStream final : public RemoteDorisStream { } private: + bool _is_cancelled() const { + return (_runtime_state != nullptr && _runtime_state->is_cancelled()) || + (_io_ctx != nullptr && _io_ctx->should_stop); + } + + void _cancel_flight_call() { + std::lock_guard lock(_flight_mutex); + if (_stream != nullptr) { + _stream->Cancel(); + } + } + + void _watch_cancellation_loop(std::stop_token watcher_stop_token) { + while (!watcher_stop_token.stop_requested()) { + if (_is_cancelled()) { + _cancel_flight_call(); + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + } + } + + void _watch_cancellation(std::stop_token watcher_stop_token) { + if (_runtime_state != nullptr && _runtime_state->get_query_ctx() != nullptr && + _runtime_state->get_query_ctx()->resource_ctx() != nullptr) { + // The watcher is query-owned and may allocate in Arrow while signalling cancellation. + SCOPED_ATTACH_TASK(_runtime_state); + _watch_cancellation_loop(watcher_stop_token); + return; + } + // Metadata/tests can construct a RuntimeState without a QueryContext; initialize TLS there + // instead of violating AttachTask's non-null resource-context invariant. + SCOPED_INIT_THREAD_CONTEXT(); + _watch_cancellation_loop(watcher_stop_token); + } + + void _stop_cancellation_watcher() { + if (_cancellation_watcher.joinable()) { + _cancellation_watcher.request_stop(); + _cancellation_watcher.join(); + } + } + const TFileRangeDesc _range; + std::shared_ptr _io_ctx; + RuntimeState* _runtime_state; + int _timeout_seconds; + std::jthread _cancellation_watcher; + std::mutex _flight_mutex; std::unique_ptr _flight_client; std::unique_ptr _stream; }; -Status create_flight_stream(const TFileRangeDesc& range, std::unique_ptr* out) { +Status create_flight_stream(const TFileRangeDesc& range, std::shared_ptr io_ctx, + RuntimeState* runtime_state, int timeout_seconds, + std::unique_ptr* out) { DORIS_CHECK(out != nullptr); - auto stream = std::make_unique(range); + auto stream = std::make_unique(range, std::move(io_ctx), runtime_state, + timeout_seconds); RETURN_IF_ERROR(stream->open()); *out = std::move(stream); return Status::OK(); @@ -177,8 +337,31 @@ RemoteDorisFileReader::~RemoteDorisFileReader() { static_cast(close()); } +void RemoteDorisFileReader::_init_profile() { + if (_profile == nullptr) { + return; + } + const auto hierarchy = file_scan_profile::ensure_hierarchy(_profile); + _io_time = hierarchy.io; + static const char* remote_profile = "RemoteDorisFileReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, remote_profile, file_scan_profile::FILE_READER, 1); + _open_stream_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisOpenStreamTime", remote_profile, 1); + _next_batch_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisNextBatchTime", remote_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisMaterializeTime", remote_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisFilterTime", remote_profile, 1); +} + Status RemoteDorisFileReader::init(RuntimeState* state) { - (void)state; + _init_profile(); + SCOPED_TIMER(_total_time); + if (state != nullptr) { + _flight_timeout_seconds = std::max(1, state->execution_timeout()); + } + _runtime_state = state; RETURN_IF_ERROR(validate_remote_doris_range(_range)); RETURN_IF_ERROR(_build_col_name_to_file_id()); _eof = false; @@ -186,6 +369,7 @@ Status RemoteDorisFileReader::init(RuntimeState* state) { } Status RemoteDorisFileReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_schema != nullptr); file_schema->clear(); file_schema->reserve(_file_slot_descs.size()); @@ -206,6 +390,8 @@ Status RemoteDorisFileReader::get_schema(std::vector* file_sch } Status RemoteDorisFileReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_stream_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); RETURN_IF_ERROR(_open_stream()); _eof = false; @@ -213,31 +399,51 @@ Status RemoteDorisFileReader::open(std::shared_ptr request) { } Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); if (_stream == nullptr) { return Status::InternalError("Remote Doris v2 reader is not open"); } + if (_io_ctx != nullptr && _io_ctx->should_stop) { + // Observe cancellation before entering a potentially blocking Flight read; the production + // stream also carries a query-bounded RPC deadline for cancellation arriving mid-read. + RETURN_IF_ERROR(close()); + *rows = 0; + *eof = true; + return Status::OK(); + } *rows = 0; *eof = false; std::shared_ptr batch; - RETURN_IF_ERROR(_stream->next(&batch)); + { + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_next_batch_time); + RETURN_IF_ERROR(_stream->next(&batch)); + } if (batch == nullptr) { *eof = true; _eof = true; return Status::OK(); } - RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + } _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR( - apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR( + apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + } return Status::OK(); } Status RemoteDorisFileReader::close() { + SCOPED_TIMER(_total_time); if (_stream != nullptr) { RETURN_IF_ERROR(_stream->close()); _stream.reset(); @@ -252,7 +458,8 @@ Status RemoteDorisFileReader::_open_stream() { if (_stream_factory) { RETURN_IF_ERROR(_stream_factory(_range, &_stream)); } else { - RETURN_IF_ERROR(create_flight_stream(_range, &_stream)); + RETURN_IF_ERROR(create_flight_stream(_range, _io_ctx, _runtime_state, + _flight_timeout_seconds, &_stream)); } DORIS_CHECK(_stream != nullptr); return Status::OK(); @@ -349,7 +556,12 @@ Status RemoteDorisReader::init(TableReadOptions&& options) { } Status RemoteDorisReader::prepare_split(const SplitReadOptions& options) { - RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + { + // Keep protocol validation visible while avoiding overlap with TableReader's own scopes. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + } return TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table/remote_doris_reader.h b/be/src/format_v2/table/remote_doris_reader.h index b4dd2a505a95ad..3b4a03f4569396 100644 --- a/be/src/format_v2/table/remote_doris_reader.h +++ b/be/src/format_v2/table/remote_doris_reader.h @@ -71,6 +71,7 @@ class RemoteDorisFileReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _open_stream(); Status _materialize_record_batch(const arrow::RecordBatch& batch, Block* file_block, size_t* rows) const; @@ -83,6 +84,14 @@ class RemoteDorisFileReader final : public FileReader { const std::vector _file_slot_descs; RemoteDorisStreamFactory _stream_factory; cctz::time_zone _ctz; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_stream_time = nullptr; + RuntimeProfile::Counter* _next_batch_time = nullptr; + RuntimeProfile::Counter* _io_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + RuntimeState* _runtime_state = nullptr; + int _flight_timeout_seconds = 300; std::unique_ptr _stream; std::unordered_map _col_name_to_file_id; }; diff --git a/be/src/format_v2/table/schema_history_util.cpp b/be/src/format_v2/table/schema_history_util.cpp index 10109839e6987d..440211607d633d 100644 --- a/be/src/format_v2/table/schema_history_util.cpp +++ b/be/src/format_v2/table/schema_history_util.cpp @@ -77,6 +77,8 @@ void annotate_column_from_field(ColumnDefinition* column, const schema::external } column->name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}; + column->has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; if (!field.__isset.nestedField) { return; } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index f30a1d567cf562..a073f01f237506 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -41,6 +41,7 @@ #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" #include "format/table/iceberg_delete_file_reader_helper.h" +#include "format/table/iceberg_scan_semantics.h" #include "format/table/paimon_reader.h" #include "format_v2/column_mapper.h" #include "format_v2/delimited_text/csv_reader.h" @@ -49,6 +50,7 @@ #include "format_v2/native/native_reader.h" #include "format_v2/orc/orc_reader.h" #include "format_v2/parquet/parquet_reader.h" +#include "runtime/file_scan_profile.h" #include "storage/segment/condition_cache.h" #include "util/debug_points.h" #include "util/string_util.h" @@ -143,6 +145,93 @@ const schema::external::TField* get_field_ptr(const schema::external::TFieldPtr& return field_ptr.field_ptr.get(); } +ColumnDefinition build_schema_identity_from_external_field(const schema::external::TField& field) { + ColumnDefinition identity; + if (field.__isset.id) { + identity.identifier = Field::create_field(field.id); + } + identity.name = field.__isset.name ? field.name : ""; + identity.name_mapping = + field.__isset.name_mapping ? field.name_mapping : std::vector {}; + identity.has_name_mapping = + field.__isset.name_mapping_is_authoritative && field.name_mapping_is_authoritative; + if (!field.__isset.nestedField) { + return identity; + } + if (field.nestedField.__isset.struct_field && field.nestedField.struct_field.__isset.fields) { + for (const auto& child_ptr : field.nestedField.struct_field.fields) { + if (const auto* child = get_field_ptr(child_ptr); child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + } + } + } else if (field.nestedField.__isset.array_field && + field.nestedField.array_field.__isset.item_field) { + if (const auto* child = get_field_ptr(field.nestedField.array_field.item_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "element"; + } + } else if (field.nestedField.__isset.map_field) { + if (field.nestedField.map_field.__isset.key_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.key_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "key"; + } + } + if (field.nestedField.map_field.__isset.value_field) { + if (const auto* child = get_field_ptr(field.nestedField.map_field.value_field); + child != nullptr) { + identity.children.push_back(build_schema_identity_from_external_field(*child)); + identity.children.back().name = "value"; + } + } + } + return identity; +} + +const ColumnDefinition* find_identity_child(const ColumnDefinition& projected_child, + const ColumnDefinition& identity_parent) { + const auto child_it = std::ranges::find_if( + identity_parent.children, [&](const ColumnDefinition& identity_child) { + if (projected_child.has_identifier_field_id() && + identity_child.has_identifier_field_id()) { + return projected_child.get_identifier_field_id() == + identity_child.get_identifier_field_id(); + } + if (to_lower(projected_child.name) == to_lower(identity_child.name)) { + return true; + } + return std::ranges::any_of( + identity_child.name_mapping, [&](const std::string& alias) { + return to_lower(projected_child.name) == to_lower(alias); + }); + }); + return child_it == identity_parent.children.end() ? nullptr : &*child_it; +} + +void attach_full_schema_identity(ColumnDefinition* projected, const ColumnDefinition& identity) { + DORIS_CHECK(projected != nullptr); + // Access-path children control materialization, but wrapper discovery needs sibling IDs that + // were pruned from that projection. Keep the complete identity tree on a separate channel. + projected->identity_children = identity.children; + for (auto& projected_child : projected->children) { + if (const auto* identity_child = find_identity_child(projected_child, identity); + identity_child != nullptr) { + attach_full_schema_identity(&projected_child, *identity_child); + } + } +} + +void clear_initial_default_metadata(ColumnDefinition* column) { + DORIS_CHECK(column != nullptr); + column->initial_default_value.reset(); + column->initial_default_value_is_base64 = false; + for (auto& child : column->children) { + clear_initial_default_metadata(&child); + } +} + bool external_field_matches_name(const schema::external::TField& field, const std::string& name) { if (field.__isset.name && to_lower(field.name) == to_lower(name)) { return true; @@ -189,6 +278,8 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: .name = field.__isset.name ? field.name : "", .name_mapping = field.__isset.name_mapping ? field.name_mapping : std::vector {}, + .has_name_mapping = field.__isset.name_mapping_is_authoritative && + field.name_mapping_is_authoritative, .type = std::move(type), .children = {}, .default_expr = nullptr, @@ -298,12 +389,32 @@ const schema::external::TField* find_external_root_field(const TFileScanRangePar if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { return nullptr; } + if (!supports_iceberg_scan_semantics_v1(params)) { + // Old BEs used one ordered current-name/alias pass. Preserve that result for old-FE plans + // until the explicit scan-semantics marker makes exact-name precedence cluster-wide. + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && external_field_matches_name(*field, column.name)) { + return field; + } + } + return nullptr; + } + // A reused name identifies the newly added field, not an older sibling that retained that + // spelling as an alias. Exhaust exact current names before consulting historical aliases. for (const auto& field_ptr : schema->root_field.fields) { const auto* field = get_field_ptr(field_ptr); - if (field == nullptr) { - continue; + if (field != nullptr && field->__isset.name && + to_lower(field->name) == to_lower(column.name)) { + return field; } - if (external_field_matches_name(*field, column.name)) { + } + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && field->__isset.name_mapping && + std::ranges::any_of(field->name_mapping, [&](const std::string& alias) { + return to_lower(alias) == to_lower(column.name); + })) { return field; } } @@ -486,8 +597,20 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info return Status::OK(); } context->schema_column = build_schema_column_from_external_field(*schema_field, column->type); + const bool use_current_semantics = supports_iceberg_scan_semantics_v1(context->scan_params); + if (!use_current_semantics) { + // IDs and encoded defaults predate the result-changing semantics. Strip only the new + // default channel so an old-FE plan keeps the same generic root/nested values on every BE. + clear_initial_default_metadata(&*context->schema_column); + } column->identifier = context->schema_column->identifier; column->name_mapping = context->schema_column->name_mapping; + column->has_name_mapping = context->schema_column->has_name_mapping; + // Projected roots already carry a generic FE default expression, but Iceberg binary defaults + // need the raw Base64 marker so missing-file materialization can decode rather than copy text. + column->initial_default_value = context->schema_column->initial_default_value; + column->initial_default_value_is_base64 = + context->schema_column->initial_default_value_is_base64; return Status::OK(); } @@ -520,22 +643,15 @@ std::optional TableReader::_find_current_table_column_by_field } Status TableReader::init(TableReadOptions&& options) { - _scan_params = options.scan_params; - _format = options.format; - _io_ctx = options.io_ctx; - _runtime_state = options.runtime_state; _scanner_profile = options.scanner_profile; - _file_slot_descs = options.file_slot_descs; - _push_down_agg_type = options.push_down_agg_type; - _condition_cache_digest = options.condition_cache_digest; - _projected_columns = std::move(options.projected_columns); - _system_properties = create_system_properties(_scan_params); - _mapper_options.mode = TableColumnMappingMode::BY_NAME; - _conjuncts = std::move(options.conjuncts); - if (_scanner_profile != nullptr) { - static const char* table_profile = "TableReader"; - ADD_TIMER_WITH_LEVEL(_scanner_profile, table_profile, 1); + const auto hierarchy = file_scan_profile::ensure_hierarchy(_scanner_profile); + static const char* table_profile = file_scan_profile::TABLE_READER; + static const char* file_reader_profile = file_scan_profile::FILE_READER; + _profile.total_timer = hierarchy.table_reader; + _profile.file_reader_total_timer = hierarchy.file_reader; + _profile.init_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "InitTime", table_profile, 1); _profile.num_delete_files = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteFiles", TUnit::UNIT, table_profile, 1); _profile.num_delete_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteRows", @@ -562,16 +678,124 @@ Status TableReader::init(TableReadOptions&& options) { ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PrepareSplitTime", table_profile, 1); _profile.finalize_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "FinalizeBlockTime", table_profile, 1); + _profile.residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "ResidualFilterTime", table_profile, 1); _profile.create_reader_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CreateReaderTime", table_profile, 1); _profile.pushdown_agg_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PushDownAggTime", table_profile, 1); _profile.open_reader_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "OpenReaderTime", table_profile, 1); - _profile.runtime_filter_partition_prune_timer = ADD_TIMER_WITH_LEVEL( - _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", 1); - _profile.runtime_filter_partition_pruned_range_counter = ADD_COUNTER_WITH_LEVEL( - _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, 1); + _profile.runtime_filter_partition_prune_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", table_profile, 1); + _profile.runtime_filter_partition_pruned_range_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, + table_profile, 1); + _profile.close_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CloseTime", table_profile, 1); + // Lifecycle timer names remain globally unique because RuntimeProfile's visual hierarchy + // does not namespace counters that share the same display parent. + _profile.file_reader_init_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderInitTime", file_reader_profile, 1); + _profile.file_reader_schema_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetSchemaTime", file_reader_profile, 1); + _profile.file_reader_mapper_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCreateColumnMapperTime", file_reader_profile, 1); + _profile.file_reader_open_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderOpenTime", file_reader_profile, 1); + _profile.file_reader_get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetBlockTime", file_reader_profile, 1); + _profile.file_reader_aggregate_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderAggregatePushDownTime", file_reader_profile, 1); + _profile.file_reader_close_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCloseTime", file_reader_profile, 1); + } + // Establish lifecycle timers before consuming options or constructing filesystem properties; + // placing these scopes at the tail records only scope teardown and hides expensive init work. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + _scan_params = options.scan_params; + _format = options.format; + _io_ctx = options.io_ctx; + _runtime_state = options.runtime_state; + _file_slot_descs = options.file_slot_descs; + _push_down_agg_type = options.push_down_agg_type; + _push_down_count_columns = options.push_down_count_columns; + _initial_condition_cache_digest = options.condition_cache_digest; + _condition_cache_digest = _initial_condition_cache_digest; + _table_reader_owned_conjunct_count = + options.table_reader_owned_conjunct_count.value_or(options.conjuncts.size()); + DORIS_CHECK_LE(_table_reader_owned_conjunct_count, options.conjuncts.size()); + _projected_columns = std::move(options.projected_columns); + if (supports_iceberg_scan_semantics_v1(_scan_params)) { + for (auto& projected_column : _projected_columns) { + const auto* schema_field = find_external_root_field(_scan_params, projected_column); + if (schema_field != nullptr) { + attach_full_schema_identity( + &projected_column, + build_schema_identity_from_external_field(*schema_field)); + } + } + } + _system_properties = create_system_properties(_scan_params); + _mapper_options.mode = TableColumnMappingMode::BY_NAME; + return _replace_conjuncts(options.conjuncts); +} + +Status TableReader::_prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared) { + DORIS_CHECK(source != nullptr); + DORIS_CHECK(source->root() != nullptr); + DORIS_CHECK(prepared != nullptr); + VExprSPtr root; + RETURN_IF_ERROR(clone_table_expr_tree(source->root(), &root)); + auto conjunct = VExprContext::create_shared(std::move(root)); + RETURN_IF_ERROR(conjunct->prepare(_runtime_state, RowDescriptor {})); + RETURN_IF_ERROR(conjunct->open(_runtime_state)); + *prepared = std::move(conjunct); + return Status::OK(); +} + +Status TableReader::_replace_conjuncts(const VExprContextSPtrs& conjuncts) { + VExprContextSPtrs prepared; + prepared.reserve(conjuncts.size()); + for (const auto& source : conjuncts) { + VExprContextSPtr conjunct; + RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + prepared.push_back(std::move(conjunct)); + } + _conjuncts = std::move(prepared); + return Status::OK(); +} + +Status TableReader::append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, + size_t table_reader_owned_conjunct_count) { + DORIS_CHECK(!_appended_table_reader_owned_conjunct_count.has_value()); + _appended_table_reader_owned_conjunct_count = table_reader_owned_conjunct_count; + auto status = append_conjuncts(conjuncts); + _appended_table_reader_owned_conjunct_count.reset(); + return status; +} + +Status TableReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + DORIS_CHECK_LE(owned_count, conjuncts.size()); + // Once Scanner owns a suffix, later predicates cannot be inserted into the TableReader-owned + // prefix without reordering them ahead of that stateful/error-preserving barrier. + DORIS_CHECK(owned_count == 0 || _table_reader_owned_conjunct_count == _conjuncts.size()); + for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { + const auto& source = conjuncts[conjunct_index]; + VExprContextSPtr conjunct; + RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + _conjuncts.push_back(conjunct); + if (conjunct_index < owned_count) { + ++_table_reader_owned_conjunct_count; + } + if (_current_task != nullptr && conjunct_index < owned_count) { + // The active reader has already fixed its localized predicate set. Appended runtime + // filters must remain residual until the next split rebuilds its FileScanRequest. + _remaining_conjuncts.push_back(std::move(conjunct)); + } } return Status::OK(); } @@ -580,22 +804,83 @@ Status TableReader::_build_table_filters_from_conjuncts() { _table_filters.clear(); _constant_pruning_safe_filter_count = 0; bool in_safe_prefix = true; - for (const auto& conjunct : _conjuncts) { + for (size_t conjunct_index = 0; conjunct_index < _conjuncts.size(); ++conjunct_index) { + const auto& conjunct = _conjuncts[conjunct_index]; DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); // `_table_filters` omits expressions without slot references, but such an expression still // occupies a position in the row-level conjunct order. Record how many localized filters // precede the first unsafe original conjunct so constant pruning cannot jump over a - // slotless non-deterministic/error-preserving barrier. - if (in_safe_prefix && !_is_safe_to_pre_execute(conjunct)) { + // slotless non-deterministic/error-preserving barrier. An unsafe predicate is either kept + // on TableReader's post-materialization path by a standalone caller or carried only for + // analysis when FileScannerV2 owns the ordered suffix. + if (in_safe_prefix && !is_safe_to_pre_execute(conjunct)) { in_safe_prefix = false; } - if (!in_safe_prefix) { - continue; - } + const size_t filters_before = _table_filters.size(); RETURN_IF_ERROR( build_table_filters_from_conjunct(conjunct, _runtime_state, &_table_filters)); - _constant_pruning_safe_filter_count = _table_filters.size(); + for (size_t filter_index = filters_before; filter_index < _table_filters.size(); + ++filter_index) { + _table_filters[filter_index].source_conjunct_index = conjunct_index; + _table_filters[filter_index].can_localize = in_safe_prefix; + } + if (in_safe_prefix) { + _constant_pruning_safe_filter_count = _table_filters.size(); + } + } + return Status::OK(); +} + +Status TableReader::_prepare_all_conjuncts_as_remaining() { + // Expression contexts carry mutable state (for example sequence/stateful functions). Select + // from the TableReader-owned contexts instead of reopening clones for every split. + _remaining_conjuncts.assign( + _conjuncts.begin(), + _conjuncts.begin() + cast_set(_table_reader_owned_conjunct_count)); + return Status::OK(); +} + +Status TableReader::_prepare_remaining_conjuncts( + const FilterLocalizationResult& localization_result) { + DORIS_CHECK(localization_result.localized_filters.size() == _table_filters.size()); + std::vector localized_conjuncts(_conjuncts.size(), false); + for (size_t filter_index = 0; filter_index < _table_filters.size(); ++filter_index) { + if (!localization_result.localized_filters[filter_index]) { + continue; + } + const size_t source_index = _table_filters[filter_index].source_conjunct_index; + DORIS_CHECK(source_index < localized_conjuncts.size()); + localized_conjuncts[source_index] = true; + } + + _remaining_conjuncts.clear(); + for (size_t conjunct_index = 0; conjunct_index < _table_reader_owned_conjunct_count; + ++conjunct_index) { + if (localized_conjuncts[conjunct_index]) { + continue; + } + _remaining_conjuncts.push_back(_conjuncts[conjunct_index]); + } + return Status::OK(); +} + +Status TableReader::_filter_remaining_conjuncts(Block* block, size_t* rows) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(rows != nullptr); + if (*rows == 0 || _remaining_conjuncts.empty()) { + return Status::OK(); + } + SCOPED_TIMER(_profile.residual_filter_timer); + const size_t rows_before_filter = *rows; + auto status = VExprContext::filter_block(_remaining_conjuncts, block, block->columns()); + if (!status.ok() && _format == FileFormat::ORC) { + status.prepend("Orc row reader nextBatch failed. reason = "); + } + RETURN_IF_ERROR(status); + *rows = block->columns() == 0 ? rows_before_filter : block->rows(); + if (_io_ctx != nullptr) { + _io_ctx->predicate_filtered_rows += rows_before_filter - *rows; } return Status::OK(); } @@ -631,10 +916,11 @@ bool TableReader::_should_enable_condition_cache(const FileScanRequest& file_req !file_request.delete_conjuncts.empty()) { return false; } - // Runtime filters can arrive late and their payload is not guaranteed to be represented by the - // scan-local digest. Without a read-only mode, a MISS could insert a bitmap for P AND RF under - // the digest for only P. This mirrors the old FileScanner guard. - return !contains_runtime_filter(file_request.conjuncts); + // Only scanner-driven splits provide a digest rebuilt from the exact RF snapshot. Keep the + // conservative behavior for standalone TableReader callers: their initial digest may describe + // only static predicate P and must not store P AND RF under that key. + return _condition_cache_digest_covers_current_split || + !contains_runtime_filter(file_request.conjuncts); } Status TableReader::_init_reader_condition_cache(const FileScanRequest& file_request) { @@ -720,7 +1006,12 @@ Status TableReader::create_next_reader(bool* eos) { if (_batch_size > 0) { _data_reader.reader->set_batch_size(_batch_size); } - Status st = _data_reader.reader->init(_runtime_state); + Status st; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_init_timer); + st = _data_reader.reader->init(_runtime_state); + } if (!st.ok()) { if (_io_ctx != nullptr && _io_ctx->should_stop && st.is()) { *eos = true; @@ -751,10 +1042,16 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { const bool enable_mapping_timestamp_tz = _scan_params != nullptr && _scan_params->__isset.enable_mapping_timestamp_tz && _scan_params->enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = _scan_params != nullptr && + _scan_params->__isset.enable_mapping_varbinary && + _scan_params->enable_mapping_varbinary; if (_format == FileFormat::PARQUET) { + // V2 must honor the scan contract directly; otherwise Hive STRING columns backed by an + // unannotated BYTE_ARRAY are silently exposed as VARBINARY and predicate bytes no longer + // match the table type. *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); return Status::OK(); } if (_format == FileFormat::ORC) { @@ -825,11 +1122,26 @@ std::unique_ptr create_file_description(const TFileRangeDes } Status TableReader::prepare_split(const SplitReadOptions& options) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.prepare_split_timer); _current_split_pruned = false; + // Predicate localization belongs to the physical schema of one split. Clear the previous + // ownership before any early return so a pruned or failed split cannot leak it to the next one. + _remaining_conjuncts.clear(); _all_runtime_filters_applied_for_split = options.all_runtime_filters_applied; + _condition_cache_digest_covers_current_split = options.condition_cache_digest.has_value(); + if (options.condition_cache_digest.has_value()) { + // The split snapshot may include RFs that arrived after TableReader::init(). Use the digest + // computed from that exact snapshot. Example: an initial P digest must not be used to store + // the bitmap for P AND late RF{7, 9}; the scanner supplies digest(P AND RF{7, 9}) here. + _condition_cache_digest = *options.condition_cache_digest; + } else { + // An explicit scanner digest is split-scoped. Restore the init-time digest when a later + // standalone split omits it instead of leaking the previous split's RF payload into its key. + _condition_cache_digest = _initial_condition_cache_digest; + } if (options.conjuncts.has_value()) { - _conjuncts = *options.conjuncts; + RETURN_IF_ERROR(_replace_conjuncts(*options.conjuncts)); } // Update to current split format to handle ORC/PARQUET files in one table. _format = options.current_split_format; @@ -848,6 +1160,8 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { _deletion_vector = nullptr; _aggregate_pushdown_tried = false; _remaining_table_level_count = -1; + _remaining_file_level_count = -1; + _current_split_uses_metadata_count = false; _current_reader_reached_eof = false; RETURN_IF_ERROR(_evaluate_partition_prune_conjuncts(options.partition_prune_conjuncts, &_current_split_pruned)); @@ -862,12 +1176,18 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { // active and no predicate can arrive later. The metadata path can return several batches for // one split; after its first synthetic batch there is no way to recover the real rows if a // runtime filter arrives before the next scheduler turn. - if (_push_down_agg_type == TPushAggOp::type::COUNT && options.all_runtime_filters_applied && + // Table-level metadata only contains the number of rows; it cannot evaluate an expression or + // the NULL state of a COUNT argument. Require the new FE's explicit empty argument list, which + // means COUNT(*)/COUNT(1). A non-empty list means COUNT(col), while nullopt comes from an old FE + // whose COUNT semantics are unknown during a BE-first rolling upgrade. + if (_push_down_agg_type == TPushAggOp::type::COUNT && _push_down_count_columns.has_value() && + _push_down_count_columns->empty() && options.all_runtime_filters_applied && _conjuncts.empty() && options.current_range.__isset.table_format_params && options.current_range.table_format_params.__isset.table_level_row_count) { DORIS_CHECK(options.current_range.table_format_params.table_level_row_count >= -1); _remaining_table_level_count = options.current_range.table_format_params.table_level_row_count; + _current_split_uses_metadata_count = _is_table_level_count_active(); } if (_is_table_level_count_active()) { return Status::OK(); @@ -891,7 +1211,7 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& // Keep only the safe prefix of the original conjunct order. If an unsafe conjunct is // skipped, a later predicate could prune the split before the unsafe one reaches its // normal row-level evaluation point. - if (!_is_safe_to_pre_execute(conjunct)) { + if (!is_safe_to_pre_execute(conjunct)) { break; } std::set global_indices; @@ -927,7 +1247,7 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& can_filter_all); } -bool TableReader::_is_safe_to_pre_execute(const VExprContextSPtr& conjunct) { +bool TableReader::is_safe_to_pre_execute(const VExprContextSPtr& conjunct) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); const auto root = conjunct->root(); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 8d99195c21da60..48e3d996ce22fd 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -78,10 +78,14 @@ namespace doris::format { using DeleteRows = std::vector; // Row-level predicates on table/global schema. They are rewritten to file-local expressions when -// possible, and remain the source of row-level filtering after localization. +// possible; otherwise TableReader evaluates them after final table-schema materialization. struct TableFilter { VExprContextSPtr conjunct; std::vector global_indices; + size_t source_conjunct_index = 0; + // False after the first unsafe source conjunct so file-local execution cannot reorder a later + // predicate ahead of stateful or error-preserving table semantics. + bool can_localize = true; }; struct ScanTask { @@ -99,6 +103,8 @@ struct ProjectedColumnBuildContext { }; struct ReadProfile { + RuntimeProfile::Counter* total_timer = nullptr; + RuntimeProfile::Counter* init_timer = nullptr; RuntimeProfile::Counter* num_delete_files = nullptr; RuntimeProfile::Counter* num_delete_rows = nullptr; RuntimeProfile::Counter* parse_delete_file_time = nullptr; @@ -110,11 +116,28 @@ struct ReadProfile { RuntimeProfile::Counter* exec_timer = nullptr; RuntimeProfile::Counter* prepare_split_timer = nullptr; RuntimeProfile::Counter* finalize_timer = nullptr; + RuntimeProfile::Counter* residual_filter_timer = nullptr; RuntimeProfile::Counter* create_reader_timer = nullptr; RuntimeProfile::Counter* pushdown_agg_timer = nullptr; RuntimeProfile::Counter* open_reader_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_prune_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_pruned_range_counter = nullptr; + RuntimeProfile::Counter* close_timer = nullptr; + RuntimeProfile::Counter* file_reader_total_timer = nullptr; + RuntimeProfile::Counter* file_reader_init_timer = nullptr; + RuntimeProfile::Counter* file_reader_schema_timer = nullptr; + RuntimeProfile::Counter* file_reader_mapper_timer = nullptr; + RuntimeProfile::Counter* file_reader_open_timer = nullptr; + RuntimeProfile::Counter* file_reader_get_block_timer = nullptr; + RuntimeProfile::Counter* file_reader_aggregate_timer = nullptr; + RuntimeProfile::Counter* file_reader_close_timer = nullptr; +}; + +struct MaterializedBlockStats { + bool has_materialized_input = false; + size_t rows = 0; + size_t bytes = 0; + size_t allocated_bytes = 0; }; struct TableReadOptions { @@ -123,6 +146,10 @@ struct TableReadOptions { const std::vector projected_columns; // All complex conjuncts from scan operator const VExprContextSPtrs conjuncts; + // Number of leading conjuncts whose row-level execution is owned by TableReader/FileReader. + // FileScannerV2 still passes the complete ordered list so mapping, pruning guards, aggregate + // eligibility, and condition-cache analysis see the exact query semantics. nullopt means all. + const std::optional table_reader_owned_conjunct_count = std::nullopt; // File format of the underlying data files, needed for reader initialization and reader-level // filter pushdown. const FileFormat format; @@ -135,7 +162,13 @@ struct TableReadOptions { const std::vector* file_slot_descs = nullptr; // Push-down aggregate type. const TPushAggOp::type push_down_agg_type = TPushAggOp::type::NONE; - // Digest of stable pushed-down predicates. A zero digest disables condition cache. + // Table/global indices of explicit COUNT arguments. nullopt means an old FE did not send the + // semantic argument field, while an explicit empty vector means COUNT(*)/COUNT(1). Keeping + // those states separate prevents a rolling-upgrade plan from being reinterpreted by a new BE. + const std::optional> push_down_count_columns = std::nullopt; + // Initial digest of predicates available during scanner open. Scanner-driven splits override it + // with SplitReadOptions::condition_cache_digest after collecting late-arrival runtime filters. + // A zero digest disables condition cache. uint64_t condition_cache_digest = 0; }; @@ -145,7 +178,7 @@ struct SplitReadOptions { // Latest scanner conjuncts rewritten to table/global column indices. Runtime filters may // arrive after TableReader::init(), so scanner-driven splits replace the initial snapshot. // nullopt preserves the initial snapshot for standalone TableReader callers. - std::optional conjuncts; + std::optional conjuncts = std::nullopt; // Independent clones used for partition pruning because evaluation prepares and opens them // against a synthetic partition block before the file reader opens its row-level conjuncts. VExprContextSPtrs partition_prune_conjuncts; @@ -154,6 +187,10 @@ struct SplitReadOptions { // filter could arrive after synthetic rows have already been returned and those rows cannot be // retracted. Standalone TableReader callers have no scanner runtime-filter lifecycle. bool all_runtime_filters_applied = true; + // Digest for the exact scanner conjunct snapshot attached to this split. FileScannerV2 rebuilds + // it after collecting late-arrival RFs, so different RF payloads cannot share a cache entry. A + // zero value explicitly disables condition cache for this split. + std::optional condition_cache_digest; ShardedKVCache* cache = nullptr; TFileRangeDesc current_range; FileFormat current_split_format = FileFormat::PARQUET; @@ -185,6 +222,11 @@ class TableReader { #ifdef BE_TEST size_t TEST_batch_size() const { return _batch_size; } + size_t TEST_conjunct_count() const { return _conjuncts.size(); } + size_t TEST_table_reader_owned_conjunct_count() const { + return _table_reader_owned_conjunct_count; + } + void TEST_set_condition_cache_hit_count(int64_t hits) { _condition_cache_hit_count = hits; } bool TEST_current_data_file_is_immutable() const { DORIS_CHECK(_current_task != nullptr); DORIS_CHECK(_current_task->data_file != nullptr); @@ -201,11 +243,37 @@ class TableReader { virtual Status prepare_split(const SplitReadOptions& options); virtual bool current_split_pruned() const { return _current_split_pruned; } + virtual bool current_split_uses_metadata_count() const { + return _current_split_uses_metadata_count; + } + + // Runtime filters that arrive after a split has opened cannot be pushed into that file reader. + // Keep their expression contexts in TableReader and evaluate them as residual predicates for + // the active reader; later splits can localize them normally. + virtual Status append_conjuncts(const VExprContextSPtrs& conjuncts); + + // Append a full ordered snapshot delta while marking only its leading prefix as owned by + // TableReader/FileReader. This non-virtual wrapper preserves the long-standing virtual API and + // carries the ownership boundary through hybrid readers to their children. + Status append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, + size_t table_reader_owned_conjunct_count); + + // Shared safety classification for deciding which ordered conjunct prefix may execute below + // Scanner without changing stateful or error-preserving semantics. + static bool is_safe_to_pre_execute(const VExprContextSPtr& conjunct); + + virtual const MaterializedBlockStats& last_materialized_block_stats() const { + return _last_materialized_block_stats; + } // Discard the active split after the caller decides an error is ignorable, for example a // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start // with no concrete reader or split-local state left from the failed split. virtual Status abort_split() { + // Ignored open failures still spend time closing partially initialized readers. Include + // that recovery path in the common lifecycle profile so NOT_FOUND cannot become invisible. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader != nullptr) { RETURN_IF_ERROR(close_current_reader()); } else { @@ -214,7 +282,10 @@ class TableReader { } _delete_rows = nullptr; _remaining_table_level_count = -1; + _remaining_file_level_count = -1; + _current_split_uses_metadata_count = false; _current_split_pruned = false; + _remaining_conjuncts.clear(); return Status::OK(); } @@ -222,7 +293,9 @@ class TableReader { // advances across EOF, and closes exhausted readers. Subclasses provide protected hooks for // table-format-specific behavior. virtual Status get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); + _last_materialized_block_stats = {}; DORIS_CHECK(block->columns() == _projected_columns.size()); block->clear_column_data(_projected_columns.size()); @@ -239,6 +312,10 @@ class TableReader { RETURN_IF_ERROR(_read_table_level_count(block, eos)); return Status::OK(); } + if (_is_file_level_count_active()) { + RETURN_IF_ERROR(_read_file_level_count(block, eos)); + return Status::OK(); + } RETURN_IF_ERROR(create_next_reader(eos)); if (!_data_reader.reader) { DCHECK(*eos); @@ -271,8 +348,12 @@ class TableReader { _data_reader.block_template.clear_column_data( cast_set(_data_reader.file_block_layout.size())); size_t current_rows = 0; - RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, - ¤t_rows, ¤t_eof)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_get_block_timer); + RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, + ¤t_rows, ¤t_eof)); + } const bool stopped_during_read = _io_ctx != nullptr && _io_ctx->should_stop; if (current_rows == 0) { if (current_eof) { @@ -287,7 +368,7 @@ class TableReader { RETURN_IF_ERROR(_check_file_block_columns("after file reader get_block", current_rows)); #endif DORIS_CHECK(block->columns() == _data_reader.column_mapper->mappings().size()); - RETURN_IF_ERROR(finalize_chunk(block, current_rows)); + RETURN_IF_ERROR(finalize_chunk(block, ¤t_rows)); #ifndef NDEBUG RETURN_IF_ERROR( _check_table_block_columns("after finalize_chunk", block, current_rows)); @@ -296,6 +377,13 @@ class TableReader { _current_reader_reached_eof = !stopped_during_read; RETURN_IF_ERROR(close_current_reader()); } + if (current_rows == 0) { + // One materialized batch is one Scanner progress unit even when residual + // predicates reject every row. Returning here preserves row-budget and + // cancellation checks in Scanner::get_block(). + block->clear_column_data(_projected_columns.size()); + return Status::OK(); + } return Status::OK(); } } @@ -303,16 +391,21 @@ class TableReader { // Close the table reader and the currently active file reader. Subclasses that hold additional // table-format resources should override this and call TableReader::close() first. virtual Status close() { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader) { RETURN_IF_ERROR(close_current_reader()); } _current_task.reset(); _current_file_description.reset(); _remaining_table_level_count = -1; + _remaining_file_level_count = -1; + _current_split_uses_metadata_count = false; + _remaining_conjuncts.clear(); return Status::OK(); } - int64_t condition_cache_hit_count() const { return _condition_cache_hit_count; } + virtual int64_t condition_cache_hit_count() const { return _condition_cache_hit_count; } virtual std::string debug_string() const; @@ -354,6 +447,7 @@ class TableReader { Status create_next_reader(bool* eos); virtual Status create_file_reader(std::unique_ptr* reader); virtual TableColumnMappingMode mapping_mode() const { return TableColumnMappingMode::BY_NAME; } + virtual void configure_mapper_options(TableColumnMapperOptions*) const {} virtual Status annotate_file_schema(std::vector* file_schema) { DORIS_CHECK(file_schema != nullptr); return Status::OK(); @@ -364,14 +458,23 @@ class TableReader { SCOPED_TIMER(_profile.open_reader_timer); // 1. Get file schema and create column mapping. std::vector file_schema; - RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_schema_timer); + RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + } // For Paimon/Hudi, FE can provide field ids through `history_schema_info`. Annotate the // file schema before column mapping when the table format maps columns by field id. RETURN_IF_ERROR(annotate_file_schema(&file_schema)); _data_reader.file_schema = file_schema; _mapper_options.mode = mapping_mode(); + configure_mapper_options(&_mapper_options); - _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_mapper_timer); + _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + } DORIS_CHECK(_data_reader.column_mapper != nullptr); RETURN_IF_ERROR(_data_reader.column_mapper->create_mapping(_projected_columns, _partition_values, file_schema)); @@ -384,14 +487,31 @@ class TableReader { // reader with the request. File scan request carries row-level expression filters and // file-level pruning hints. Only expression filters decide returned rows. auto file_request = std::make_shared(); + FilterLocalizationResult localization_result; RETURN_IF_ERROR(_data_reader.column_mapper->create_scan_request( - _table_filters, _projected_columns, file_request.get(), _runtime_state)); + _table_filters, _projected_columns, file_request.get(), _runtime_state, + &localization_result)); bool constant_filter_pruned_split = false; RETURN_IF_ERROR(_evaluate_constant_filters(&constant_filter_pruned_split)); if (constant_filter_pruned_split) { RETURN_IF_ERROR(close_current_reader()); return Status::OK(); } + RETURN_IF_ERROR(_prepare_remaining_conjuncts(localization_result)); + // COUNT(*) has no semantic column argument, but Nereids retains a minimum-width scan slot + // so the scan node still has an output tuple. Record only the current non-predicate file + // columns before table-format hooks add row-position or equality-delete dependencies. This + // marker is independent of aggregate eligibility: with position deletes, for example, + // metadata COUNT must fall back to reading rows, but an arbitrary unsupported TIME_MILLIS + // placeholder still must not be validated or decoded merely to carry the surviving count. + if (_push_down_agg_type == TPushAggOp::type::COUNT && + _push_down_count_columns.has_value() && _push_down_count_columns->empty()) { + file_request->count_star_placeholder_columns.reserve( + file_request->non_predicate_columns.size()); + for (const auto& column : file_request->non_predicate_columns) { + file_request->count_star_placeholder_columns.push_back(column.column_id()); + } + } RETURN_IF_ERROR(customize_file_scan_request(file_request.get())); RETURN_IF_ERROR(_open_local_filter_exprs(*file_request)); _data_reader.file_block_layout.clear(); @@ -443,15 +563,23 @@ class TableReader { VLOG_DEBUG << "TableReader debug: " << debug_string(); } RETURN_IF_ERROR(_open_mapping_exprs()); - RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_open_timer); + RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + } RETURN_IF_ERROR(_init_reader_condition_cache(*file_request)); return Status::OK(); } Status _build_table_filters_from_conjuncts(); + Status _replace_conjuncts(const VExprContextSPtrs& conjuncts); + Status _prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared); + Status _prepare_remaining_conjuncts(const FilterLocalizationResult& localization_result); + Status _prepare_all_conjuncts_as_remaining(); + Status _filter_remaining_conjuncts(Block* block, size_t* rows); Status _evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts, bool* can_filter_all); - static bool _is_safe_to_pre_execute(const VExprContextSPtr& conjunct); Status _build_partition_prune_block(Block* block) const; Status _open_local_filter_exprs(const FileScanRequest& file_request); Status _init_reader_condition_cache(const FileScanRequest& file_request); @@ -470,7 +598,7 @@ class TableReader { if (table_filter.conjunct == nullptr) { continue; } - DORIS_CHECK(_is_safe_to_pre_execute(table_filter.conjunct)); + DORIS_CHECK(is_safe_to_pre_execute(table_filter.conjunct)); // RuntimeFilterExpr does not implement execute_column_impl(); it is evaluated by the // row-level filter path through execute_filter(). Constant split pruning uses // VExprContext::execute() on a one-row synthetic block, so runtime filters must not be @@ -566,39 +694,64 @@ class TableReader { bool _is_table_level_count_active() const { return _remaining_table_level_count >= 0; } + bool _is_file_level_count_active() const { return _remaining_file_level_count >= 0; } + Status _materialize_count_rows(size_t rows, Block* block) const { DORIS_CHECK(block != nullptr); DORIS_CHECK(block->columns() > 0 || rows == 0); for (size_t column_idx = 0; column_idx < block->columns(); ++column_idx) { auto column = block->get_by_position(column_idx).type->create_column(); - column->resize(rows); + if (auto* nullable = check_and_get_column(*column)) { + // Metadata COUNT emits synthetic input rows for the unchanged upper aggregate. + // They must be non-NULL for COUNT(nullable_col), and constructing them explicitly + // also keeps every nullable null map boolean-valid in debug/ASAN block checks. + nullable->get_nested_column().insert_many_defaults(rows); + nullable->get_null_map_data().resize_fill(rows, 0); + } else { + column->insert_many_defaults(rows); + } block->replace_by_position(column_idx, std::move(column)); } return Status::OK(); } - Status _read_table_level_count(Block* block, bool* eos) { + Status _materialize_next_count_batch(int64_t* remaining_rows, Block* block) const { + DORIS_CHECK(remaining_rows != nullptr); + DORIS_CHECK(*remaining_rows > 0); + const int64_t batch_size = _runtime_state == nullptr + ? *remaining_rows + : static_cast(_runtime_state->batch_size()); + const auto rows = std::min(*remaining_rows, batch_size); + RETURN_IF_ERROR(_materialize_count_rows(cast_set(rows), block)); + *remaining_rows -= rows; + return Status::OK(); + } + + Status _read_count_batch(int64_t* remaining_rows, Block* block, bool* eos) { DORIS_CHECK(block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(_push_down_agg_type == TPushAggOp::type::COUNT); - DORIS_CHECK(_remaining_table_level_count >= 0); - if (_remaining_table_level_count == 0) { - _remaining_table_level_count = -1; + DORIS_CHECK(remaining_rows != nullptr); + DORIS_CHECK(*remaining_rows >= 0); + if (*remaining_rows == 0) { + *remaining_rows = -1; _current_task.reset(); *eos = true; return Status::OK(); } - - const int64_t batch_size = _runtime_state == nullptr - ? _remaining_table_level_count - : static_cast(_runtime_state->batch_size()); - const auto rows = std::min(_remaining_table_level_count, batch_size); - RETURN_IF_ERROR(_materialize_count_rows(cast_set(rows), block)); - _remaining_table_level_count -= rows; + RETURN_IF_ERROR(_materialize_next_count_batch(remaining_rows, block)); *eos = false; return Status::OK(); } + Status _read_table_level_count(Block* block, bool* eos) { + return _read_count_batch(&_remaining_table_level_count, block, eos); + } + + Status _read_file_level_count(Block* block, bool* eos) { + return _read_count_batch(&_remaining_file_level_count, block, eos); + } + void _append_file_scan_column(FileScanRequest* request, LocalColumnId column_id, std::vector* scan_columns) { DORIS_CHECK(request != nullptr); @@ -650,7 +803,11 @@ class TableReader { // close(), so it should remain idempotent. virtual Status close_current_reader() { _finalize_reader_condition_cache(); - RETURN_IF_ERROR(_data_reader.reader->close()); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_close_timer); + RETURN_IF_ERROR(_data_reader.reader->close()); + } _data_reader.reader.reset(); if (_data_reader.column_mapper != nullptr) { _data_reader.column_mapper->clear(); @@ -658,6 +815,7 @@ class TableReader { } _table_filters.clear(); _constant_pruning_safe_filter_count = 0; + _remaining_conjuncts.clear(); _data_reader.file_schema.clear(); _data_reader.file_block_layout.clear(); _data_reader.block_template.clear(); @@ -673,14 +831,28 @@ class TableReader { } } + void _reset_materialized_block_stats() { _last_materialized_block_stats = {}; } + + void _record_materialized_block_stats(const Block& block, size_t rows) { + _last_materialized_block_stats = { + .has_materialized_input = true, + .rows = rows, + .bytes = block.bytes(), + .allocated_bytes = block.allocated_bytes(), + }; + } + // Finalize file-local block to table/global schema block. - Status finalize_chunk(Block* block, const size_t rows) { + Status finalize_chunk(Block* block, size_t* rows) { + DORIS_CHECK(rows != nullptr); SCOPED_TIMER(_profile.finalize_timer); size_t idx = 0; - for (const auto& mapping : _data_reader.column_mapper->mappings()) { + const auto& mappings = _data_reader.column_mapper->mappings(); + for (const auto& mapping : mappings) { ColumnPtr column; - RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, rows, - &column)); + RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, + *rows, &column, + idx + 1 == mappings.size())); block->replace_by_position(idx, IColumn::mutate(std::move(column))); idx++; } @@ -688,7 +860,12 @@ class TableReader { // Enforce CHAR/VARCHAR length declared by the table schema after all file-to-table // materialization has finished. RETURN_IF_ERROR(_truncate_char_or_varchar_columns(block)); - return Status::OK(); + // Preserve the cost of materialization before residual predicates shrink the block. The + // scanner uses this snapshot for bounded progress and adaptive batch sizing. + _record_materialized_block_stats(*block, *rows); + // Predicate ownership is split-local: only predicates not acknowledged as exact by this + // split's FileScanRequest run here, after virtual/default/schema-evolution values exist. + return _filter_remaining_conjuncts(block, rows); } // Materialize virtual columns in the table block, such as Iceberg _row_id and @@ -852,31 +1029,7 @@ class TableReader { // - table VARCHAR(10), file STRING: truncate to 10 because STRING has no declared bound; // - table STRING, any file type: no truncation because the target has no bound. static bool _should_truncate_char_or_varchar_column(const ColumnMapping& mapping) { - if (mapping.table_type == nullptr) { - return false; - } - const auto table_type = remove_nullable(mapping.table_type); - const auto primitive_type = table_type->get_primitive_type(); - if (primitive_type != TYPE_VARCHAR && primitive_type != TYPE_CHAR) { - return false; - } - const auto target_len = assert_cast(table_type.get())->len(); - if (target_len <= 0) { - return false; - } - if (mapping.file_type == nullptr) { - return true; - } - const auto file_type = remove_nullable(mapping.file_type); - DORIS_CHECK(file_type != nullptr); - int file_len = -1; - if (file_type->get_primitive_type() == TYPE_VARCHAR || - file_type->get_primitive_type() == TYPE_CHAR || - file_type->get_primitive_type() == TYPE_STRING) { - file_len = assert_cast(file_type.get())->len(); - } - - return file_len < 0 || target_len < file_len; + return requires_char_or_varchar_truncation(mapping); } // Truncate a materialized CHAR/VARCHAR column in place by reusing the vectorized substring @@ -931,13 +1084,30 @@ class TableReader { FileAggregateRequest file_request; RETURN_IF_ERROR(_build_file_aggregate_request(_push_down_agg_type, &file_request)); FileAggregateResult file_result; - const auto status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + Status status; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_aggregate_timer); + status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + } if (status.is()) { return Status::OK(); } RETURN_IF_ERROR(status); - RETURN_IF_ERROR( - _materialize_aggregate_pushdown_rows(_push_down_agg_type, file_result, block)); + if (_push_down_agg_type == TPushAggOp::type::COUNT) { + DORIS_CHECK(file_result.count >= 0); + // The upper aggregate consumes synthetic input rows, but emitting the whole metadata + // count in one block bypasses the runtime batch contract and can allocate by file size. + // Keep the remaining cardinality as split state and expose at most one batch per call. + _remaining_file_level_count = file_result.count; + _current_split_uses_metadata_count = true; + if (_remaining_file_level_count > 0) { + RETURN_IF_ERROR(_materialize_next_count_batch(&_remaining_file_level_count, block)); + } + } else { + RETURN_IF_ERROR( + _materialize_aggregate_pushdown_rows(_push_down_agg_type, file_result, block)); + } *pushed_down = true; RETURN_IF_ERROR(close_current_reader()); return Status::OK(); @@ -956,9 +1126,8 @@ class TableReader { if (!_all_runtime_filters_applied_for_split) { return false; } - // Scanner owns the original conjunct list and evaluates it after TableReader finalizes - // rows. Even a slotless conjunct that cannot become a TableFilter must see every source - // row before an aggregate reduces the stream to synthetic COUNT/MINMAX rows. + // Even a slotless conjunct that cannot become a TableFilter must see every source row + // before an aggregate reduces the stream to synthetic COUNT/MINMAX rows. if (!_conjuncts.empty()) { return false; } @@ -973,7 +1142,31 @@ class TableReader { return false; } if (agg_type == TPushAggOp::type::COUNT) { - return true; + // Old FEs do not serialize push_down_count_slot_ids. During the supported BE-first + // rolling upgrade, nullopt therefore means "COUNT semantics are unknown", not + // COUNT(*). Fall back to reading rows until the FE explicitly sends either an empty + // list for COUNT(*) or one slot for COUNT(col). + if (!_push_down_count_columns.has_value()) { + return false; + } + // COUNT(*) needs no column metadata. COUNT(col) currently supports one direct file + // column; multiple COUNT arguments fall back to the normal scan so every upper + // aggregate receives the original rows. + if (_push_down_count_columns->empty()) { + return true; + } + if (_push_down_count_columns->size() != 1) { + return false; + } + const auto& mapping = _push_down_count_mapping(); + // Metadata COUNT skips TableReader's normal materialization path. Only a trivial + // mapping is safe: for example, a nullable Parquet INT mapped to a NOT NULL table + // BIGINT normally needs both an INT->BIGINT cast and nullability validation. Counting + // footer values directly would bypass both operations and could hide invalid data. + return mapping.file_local_id.has_value() && mapping.file_type != nullptr && + mapping.table_type != nullptr && mapping.is_trivial && + mapping.virtual_column_type == TableVirtualColumnType::INVALID && + mapping.default_expr == nullptr; } // For MIN/MAX, only support direct file-to-table column mappings. The two emitted rows // must be enough for the upper MIN/MAX aggregate without evaluating default expressions or @@ -997,6 +1190,17 @@ class TableReader { return IColumn::mutate(std::move(column)); } + static ColumnPtr _take_and_detach_block_column(Block* block, int position) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(position >= 0 && position < static_cast(block->columns())); + auto& source = block->get_by_position(position); + ColumnPtr column = source.column; + // The final mapping no longer needs the file block. Release its COW owner before mutate(), + // otherwise nested MAP/STRING columns are deep-copied and a multi-GB payload can OOM. + block->replace_by_position(position, source.type->create_column()); + return _detach_column(std::move(column)); + } + static Status _align_column_nullability(ColumnPtr* column, const DataTypePtr& table_type) { DORIS_CHECK(column != nullptr); DORIS_CHECK(column->get() != nullptr); @@ -1140,7 +1344,8 @@ class TableReader { } Status _materialize_mapping_column(const ColumnMapping& mapping, Block* current_block, - const size_t rows, ColumnPtr* column) { + const size_t rows, ColumnPtr* column, + bool take_projection_result = false) { if (!mapping.is_trivial && mapping.file_local_id.has_value() && !mapping.child_mappings.empty()) { DCHECK(mapping.projection != nullptr); @@ -1153,7 +1358,9 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), *mapping.file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; + ColumnPtr result_column = take_projection_result + ? _take_and_detach_block_column(current_block, res_id) + : current_block->get_by_position(res_id).column; RETURN_IF_ERROR( _materialize_complex_mapping_column(mapping, result_column, rows, column)); return Status::OK(); @@ -1172,8 +1379,12 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; - *column = _detach_column(std::move(result_column)); + if (take_projection_result) { + *column = _take_and_detach_block_column(current_block, res_id); + } else { + ColumnPtr result_column = current_block->get_by_position(res_id).column; + *column = _detach_column(std::move(result_column)); + } return Status::OK(); } if (mapping.default_expr != nullptr) { @@ -1313,7 +1524,10 @@ class TableReader { DORIS_CHECK(child_mapping != nullptr); if (!child_mapping->file_local_id.has_value()) { child_columns.push_back( - child_mapping->table_type->create_column_const_with_default_value(rows) + (child_mapping->initial_default_column + ? child_mapping->initial_default_column->clone_resized(rows) + : child_mapping->table_type + ->create_column_const_with_default_value(rows)) ->convert_to_full_column_if_const()); continue; } @@ -1456,24 +1670,19 @@ class TableReader { request->agg_type = agg_type; request->columns.clear(); if (agg_type == TPushAggOp::type::COUNT) { - // COUNT pushdown historically meant COUNT(*) and therefore carried no columns. For - // complex COUNT(col), materializing the full MAP/LIST/STRUCT value only to test the - // top-level NULL bit can be extremely expensive. When the scan projects exactly one - // directly-mapped complex column, pass that file column to the reader so formats such - // as Parquet can count the column shape from metadata/levels without decoding payload - // values like MAP value strings. Other COUNT cases stay on the existing row-count path - // to avoid changing count(*) semantics. - if (_data_reader.column_mapper->mappings().size() == 1) { - const auto& mapping = _data_reader.column_mapper->mappings()[0]; - if (mapping.file_local_id.has_value() && mapping.file_type != nullptr && - is_complex_type(remove_nullable(mapping.file_type)->get_primitive_type()) && - mapping.virtual_column_type == TableVirtualColumnType::INVALID && - mapping.default_expr == nullptr) { - FileAggregateRequest::Column column; - column.projection = - LocalColumnIndex::top_level(LocalColumnId(*mapping.file_local_id)); - request->columns.push_back(std::move(column)); - } + DORIS_CHECK(_push_down_count_columns.has_value()); + // An empty explicit list is the semantic signal for COUNT(*). Do not inspect the + // mapping count: `SELECT COUNT(*) FROM t` may still project one nullable column because + // the planner keeps a placeholder slot. In a 10,000-row file where that arbitrary slot + // has 9,015 non-null values, passing the slot would ask Parquet/ORC metadata for + // COUNT(slot)=9,015 instead of the required row count 10,000. + if (!_push_down_count_columns->empty()) { + const auto& mapping = _push_down_count_mapping(); + DORIS_CHECK(mapping.file_local_id.has_value()); + FileAggregateRequest::Column column; + column.projection = + LocalColumnIndex::top_level(LocalColumnId(*mapping.file_local_id)); + request->columns.push_back(std::move(column)); } return Status::OK(); } @@ -1490,16 +1699,22 @@ class TableReader { return Status::OK(); } + const ColumnMapping& _push_down_count_mapping() const { + DORIS_CHECK(_push_down_count_columns.has_value()); + DORIS_CHECK(_push_down_count_columns->size() == 1); + const auto mapping_it = + std::ranges::find(_data_reader.column_mapper->mappings(), + _push_down_count_columns->front(), &ColumnMapping::global_index); + // FileScannerV2 translates FE SlotIds through the same projected-column list used to build + // the mapper, so a missing mapping is an FE/BE contract violation rather than a fallback. + DORIS_CHECK(mapping_it != _data_reader.column_mapper->mappings().end()); + return *mapping_it; + } + Status _materialize_aggregate_pushdown_rows(TPushAggOp::type agg_type, const FileAggregateResult& file_result, Block* block) { - if (agg_type == TPushAggOp::type::COUNT) { - // COUNT pushdown is not a final count value. It emits `count` default rows so the - // upper COUNT(*) aggregate can count them and produce the final result, including - // zero rows when count is 0. - DORIS_CHECK(file_result.count >= 0); - return _materialize_count_rows(cast_set(file_result.count), block); - } + DORIS_CHECK(agg_type == TPushAggOp::type::MINMAX); // MIN/MAX pushdown emits two rows, min first and max second, for each projected column. // The upper MIN/MAX aggregate consumes those two rows to produce the final aggregate value. DORIS_CHECK(file_result.columns.size() == _data_reader.column_mapper->mappings().size()); @@ -1539,9 +1754,10 @@ class TableReader { for (size_t column_idx = 0; column_idx < _data_reader.column_mapper->mappings().size(); ++column_idx) { ColumnPtr table_column; - RETURN_IF_ERROR( - _materialize_mapping_column(_data_reader.column_mapper->mappings()[column_idx], - &file_block, 2, &table_column)); + RETURN_IF_ERROR(_materialize_mapping_column( + _data_reader.column_mapper->mappings()[column_idx], &file_block, 2, + &table_column, + column_idx + 1 == _data_reader.column_mapper->mappings().size())); block->replace_by_position(column_idx, std::move(table_column)); } return Status::OK(); @@ -1583,6 +1799,10 @@ class TableReader { // intentionally absent from that vector but must still act as ordering barriers. size_t _constant_pruning_safe_filter_count = 0; VExprContextSPtrs _conjuncts; + size_t _table_reader_owned_conjunct_count = 0; + std::optional _appended_table_reader_owned_conjunct_count; + VExprContextSPtrs _remaining_conjuncts; + MaterializedBlockStats _last_materialized_block_stats; ReadProfile _profile; // Parsed from row-position based delete files, including position delete and deletion vector. DeleteRows* _delete_rows = nullptr; @@ -1594,14 +1814,25 @@ class TableReader { const std::vector* _file_slot_descs = nullptr; FileFormat _format; TPushAggOp::type _push_down_agg_type = TPushAggOp::type::NONE; + std::optional> _push_down_count_columns; size_t _batch_size = 0; + uint64_t _initial_condition_cache_digest = 0; uint64_t _condition_cache_digest = 0; + // True only when prepare_split() received a digest for the exact conjunct snapshot used by + // this split. Standalone callers that only supplied TableReadOptions::condition_cache_digest + // keep the conservative runtime-filter guard. + bool _condition_cache_digest_covers_current_split = false; segment_v2::ConditionCache::ExternalCacheKey _condition_cache_key; std::shared_ptr> _condition_cache; std::shared_ptr _condition_cache_ctx; int64_t _condition_cache_hit_count = 0; bool _current_reader_reached_eof = false; int64_t _remaining_table_level_count = -1; + int64_t _remaining_file_level_count = -1; + // True only after the active split selects a table-level row-count shortcut or successfully + // materializes COUNT rows from file metadata. FileScannerV2 uses this result, rather than the + // raw aggregate opcode, to keep adaptive batching enabled for normal row-scan fallbacks. + bool _current_split_uses_metadata_count = false; // Snapshot supplied by FileScannerV2 for the active split. It gates every shortcut that emits // irreversible aggregate rows, not only the table-level row-count shortcut in prepare_split(). bool _all_runtime_filters_applied_for_split = true; diff --git a/be/src/information_schema/schema_routine_load_job_scanner.cpp b/be/src/information_schema/schema_routine_load_job_scanner.cpp index 9e94a507d6c98a..0e954de76466bf 100644 --- a/be/src/information_schema/schema_routine_load_job_scanner.cpp +++ b/be/src/information_schema/schema_routine_load_job_scanner.cpp @@ -55,6 +55,7 @@ std::vector SchemaRoutineLoadJobScanner::_s_tbls_colu {"CURRENT_ABORT_TASK_NUM", TYPE_INT, sizeof(int32_t), true}, {"IS_ABNORMAL_PAUSE", TYPE_BOOLEAN, sizeof(int8_t), true}, {"COMPUTE_GROUP", TYPE_STRING, sizeof(StringRef), true}, + {"FIRST_ERROR_MSG", TYPE_STRING, sizeof(StringRef), true}, }; SchemaRoutineLoadJobScanner::SchemaRoutineLoadJobScanner() @@ -175,6 +176,9 @@ Status SchemaRoutineLoadJobScanner::_fill_block_impl(Block* block) { case 20: // COMPUTE_GROUP column_value = job_info.__isset.compute_group ? job_info.compute_group : ""; break; + case 21: // FIRST_ERROR_MSG + column_value = job_info.__isset.first_error_msg ? job_info.first_error_msg : ""; + break; } str_refs[row_idx] = diff --git a/be/src/io/cache/block_file_cache_factory.cpp b/be/src/io/cache/block_file_cache_factory.cpp index 0ffc9cea365d6f..98ba9026efbf47 100644 --- a/be/src/io/cache/block_file_cache_factory.cpp +++ b/be/src/io/cache/block_file_cache_factory.cpp @@ -33,7 +33,10 @@ #include #include #include +#include #include +#include +#include #include #include "common/config.h" @@ -42,6 +45,7 @@ #include "io/cache/file_cache_common.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" +#include "runtime/thread_context.h" #include "service/backend_options.h" #include "util/slice.h" @@ -50,28 +54,16 @@ class TUniqueId; namespace io { -FileCacheFactory* FileCacheFactory::instance() { - return ExecEnv::GetInstance()->file_cache_factory(); -} +namespace { -size_t FileCacheFactory::try_release() { - int elements = 0; - for (auto& cache : _caches) { - elements += cache->try_release(); - } - return elements; -} - -size_t FileCacheFactory::try_release(const std::string& base_path) { - auto iter = _path_to_cache.find(base_path); - if (iter != _path_to_cache.end()) { - return iter->second->try_release(); - } - return 0; -} +struct BuiltFileCache { + std::string cache_base_path; + FileCacheSettings settings; + std::unique_ptr cache; +}; -Status FileCacheFactory::create_file_cache(const std::string& cache_base_path, - FileCacheSettings file_cache_settings) { +Status build_file_cache(const std::string& cache_base_path, FileCacheSettings file_cache_settings, + BuiltFileCache* built_cache) { if (file_cache_settings.storage == "memory") { if (cache_base_path != "memory") { LOG(WARNING) << "memory storage must use memory path"; @@ -113,13 +105,112 @@ Status FileCacheFactory::create_file_cache(const std::string& cache_base_path, << " total_size: " << file_cache_settings.capacity << " disk_total_size: " << disk_capacity; } + auto cache = std::make_unique(cache_base_path, file_cache_settings); RETURN_IF_ERROR(cache->initialize()); + built_cache->cache_base_path = cache_base_path; + built_cache->settings = file_cache_settings; + built_cache->cache = std::move(cache); + return Status::OK(); +} + +} // namespace + +FileCacheFactory* FileCacheFactory::instance() { + return ExecEnv::GetInstance()->file_cache_factory(); +} + +size_t FileCacheFactory::try_release() { + int elements = 0; + for (auto& cache : _caches) { + elements += cache->try_release(); + } + return elements; +} + +size_t FileCacheFactory::try_release(const std::string& base_path) { + auto iter = _path_to_cache.find(base_path); + if (iter != _path_to_cache.end()) { + return iter->second->try_release(); + } + return 0; +} + +Status FileCacheFactory::create_file_cache(const std::string& cache_base_path, + FileCacheSettings file_cache_settings) { + BuiltFileCache built_cache; + RETURN_IF_ERROR(build_file_cache(cache_base_path, file_cache_settings, &built_cache)); + { + std::lock_guard lock(_mtx); + _path_to_cache[built_cache.cache_base_path] = built_cache.cache.get(); + _capacity += built_cache.settings.capacity; + _caches.push_back(std::move(built_cache.cache)); + } + + return Status::OK(); +} + +Status FileCacheFactory::create_file_caches( + const std::vector& cache_paths, + const std::function& should_ignore_error) { + struct BuildResult { + std::string cache_base_path; + FileCacheSettings settings; + BuiltFileCache built_cache; + Status status; + bool skip = false; + }; + + std::vector results; + results.reserve(cache_paths.size()); + std::unordered_set cache_path_set; + for (const auto& cache_path : cache_paths) { + if (cache_path_set.find(cache_path.path) != cache_path_set.end()) { + LOG(WARNING) << fmt::format("cache path {} is duplicate", cache_path.path); + continue; + } + + cache_path_set.emplace(cache_path.path); + auto& result = results.emplace_back(); + result.cache_base_path = cache_path.path; + result.settings = cache_path.init_settings(); + } + + std::vector workers; + workers.reserve(results.size()); + for (auto& result : results) { + auto* result_ptr = &result; + workers.emplace_back([result_ptr]() { + SCOPED_INIT_THREAD_CONTEXT(); + result_ptr->status = build_file_cache(result_ptr->cache_base_path, result_ptr->settings, + &result_ptr->built_cache); + }); + } + + for (auto& worker : workers) { + worker.join(); + } + + for (auto& result : results) { + if (!result.status.ok()) { + if (should_ignore_error && should_ignore_error(result.cache_base_path, result.status)) { + result.skip = true; + continue; + } + return result.status; + } + } + { std::lock_guard lock(_mtx); - _path_to_cache[cache_base_path] = cache.get(); - _caches.push_back(std::move(cache)); - _capacity += file_cache_settings.capacity; + for (auto& result : results) { + if (result.skip) { + continue; + } + _path_to_cache[result.built_cache.cache_base_path] = result.built_cache.cache.get(); + _capacity += result.built_cache.settings.capacity; + _caches.push_back(std::move(result.built_cache.cache)); + } } return Status::OK(); diff --git a/be/src/io/cache/block_file_cache_factory.h b/be/src/io/cache/block_file_cache_factory.h index f5be334de8b75f..b89065b176bb2e 100644 --- a/be/src/io/cache/block_file_cache_factory.h +++ b/be/src/io/cache/block_file_cache_factory.h @@ -22,6 +22,7 @@ #include +#include #include #include #include @@ -50,6 +51,10 @@ class FileCacheFactory { Status create_file_cache(const std::string& cache_base_path, FileCacheSettings file_cache_settings); + Status create_file_caches( + const std::vector& cache_paths, + const std::function& should_ignore_error); + Status reload_file_cache(const std::vector& cache_base_paths); size_t try_release(); diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 751d422f044c1d..7aed33ccaa6e07 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -126,13 +126,26 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(segment_footer_index_bytes_write_into_cache); SUBTRACT_FIELD(remote_only_on_miss_triggered); SUBTRACT_FIELD(remote_only_on_miss_threshold_bytes); + + SUBTRACT_FIELD(num_cross_cg_peer_io_total); + SUBTRACT_FIELD(bytes_read_from_cross_cg_peer); + SUBTRACT_FIELD(cross_cg_peer_io_timer); + SUBTRACT_FIELD(num_same_cg_peer_io_total); + SUBTRACT_FIELD(bytes_read_from_same_cg_peer); + SUBTRACT_FIELD(same_cg_peer_io_timer); + SUBTRACT_FIELD(num_peer_race_peer_win); + SUBTRACT_FIELD(num_peer_race_s3_win); + SUBTRACT_FIELD(num_peer_lazy_fetch); + SUBTRACT_FIELD(peer_lazy_fetch_timer); #undef SUBTRACT_FIELD return diff; } -FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _profile(profile) { +FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, + const std::string& parent_counter) + : _profile(profile) { static const char* cache_profile = "FileCache"; - ADD_TIMER_WITH_LEVEL(profile, cache_profile, 2); + total_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, cache_profile, parent_counter.c_str(), 2); num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumLocalIOTotal", TUnit::UNIT, cache_profile, 1); num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumRemoteIOTotal", TUnit::UNIT, @@ -239,6 +252,12 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _p } void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) const { + // These are the outer cache-path phases. Their sum keeps the group timer actionable instead of + // displaying zero while individual cache IO and coordination timers are non-zero. + COUNTER_UPDATE(total_time, statistics->local_io_timer + statistics->remote_io_timer + + statistics->peer_io_timer + statistics->remote_wait_timer + + statistics->write_cache_io_timer + + statistics->cache_get_or_set_timer); COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total); COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total); COUNTER_UPDATE(num_peer_io_total, statistics->num_peer_io_total); diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index 2a6e7b33980bb2..d3fd31033649c8 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -71,6 +71,7 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren struct FileCacheProfileReporter { RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* num_local_io_total = nullptr; RuntimeProfile::Counter* num_remote_io_total = nullptr; RuntimeProfile::Counter* num_peer_io_total = nullptr; @@ -129,7 +130,9 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* num_peer_lazy_fetch = nullptr; RuntimeProfile::Counter* peer_lazy_fetch_timer = nullptr; - FileCacheProfileReporter(RuntimeProfile* profile); + explicit FileCacheProfileReporter( + RuntimeProfile* profile, + const std::string& parent_counter = RuntimeProfile::ROOT_COUNTER); void update(const FileCacheStatistics* statistics) const; }; diff --git a/be/src/io/cache/cache_lru_dumper.cpp b/be/src/io/cache/cache_lru_dumper.cpp index 43275f5069e614..289b5d49d4a9e5 100644 --- a/be/src/io/cache/cache_lru_dumper.cpp +++ b/be/src/io/cache/cache_lru_dumper.cpp @@ -393,6 +393,8 @@ Status CacheLRUDumper::parse_dump_footer(std::ifstream& in, std::string& filenam RETURN_IF_ERROR(check_ifstream_status(in, filename)); _parse_meta.Clear(); _current_parse_group.Clear(); + _parse_group_index = 0; + _parse_entry_index = 0; if (!_parse_meta.ParseFromString(meta_serialized)) { std::string warn_msg = std::string( fmt::format("LRU dump file meta parse failed, file={}, skip restore", filename)); @@ -407,13 +409,13 @@ Status CacheLRUDumper::parse_dump_footer(std::ifstream& in, std::string& filenam Status CacheLRUDumper::parse_one_lru_entry(std::ifstream& in, std::string& filename, UInt128Wrapper& hash, size_t& offset, size_t& size) { - // Read next group if current is empty - if (_current_parse_group.entries_size() == 0) { - if (_parse_meta.group_offset_size_size() == 0) { + // Read next group if current group has been fully consumed. + if (_parse_entry_index >= _current_parse_group.entries_size()) { + if (_parse_group_index >= _parse_meta.group_offset_size_size()) { return Status::EndOfFile("No more entries"); } - auto group_info = _parse_meta.group_offset_size(0); + const auto& group_info = _parse_meta.group_offset_size(_parse_group_index++); in.seekg(group_info.offset(), std::ios::beg); std::string group_serialized(group_info.size(), '\0'); in.read(&group_serialized[0], group_serialized.size()); @@ -431,20 +433,16 @@ Status CacheLRUDumper::parse_one_lru_entry(std::ifstream& in, std::string& filen LOG(WARNING) << warn_msg; return Status::InternalError(warn_msg); } - - // Remove processed group info - _parse_meta.mutable_group_offset_size()->erase(_parse_meta.group_offset_size().begin()); + _parse_entry_index = 0; } // Get next entry from current group VLOG_DEBUG << "After deserialization: " << _current_parse_group.DebugString(); - auto entry = _current_parse_group.entries(0); + const auto& entry = _current_parse_group.entries(_parse_entry_index++); hash = UInt128Wrapper((static_cast(entry.hash().high()) << 64) | entry.hash().low()); offset = entry.offset(); size = entry.size(); - // Remove processed entry - _current_parse_group.mutable_entries()->erase(_current_parse_group.entries().begin()); return Status::OK(); } diff --git a/be/src/io/cache/cache_lru_dumper.h b/be/src/io/cache/cache_lru_dumper.h index 8074ee334a1934..c4641388b55ce9 100644 --- a/be/src/io/cache/cache_lru_dumper.h +++ b/be/src/io/cache/cache_lru_dumper.h @@ -87,6 +87,8 @@ class CacheLRUDumper { // For parsing doris::io::cache::LRUDumpEntryGroupPb _current_parse_group; doris::io::cache::LRUDumpMetaPb _parse_meta; + int _parse_group_index = 0; + int _parse_entry_index = 0; BlockFileCache* _mgr; LRUQueueRecorder* _recorder; @@ -94,4 +96,4 @@ class CacheLRUDumper { std::string _start_time; bool _is_first_dump = true; }; -} // namespace doris::io \ No newline at end of file +} // namespace doris::io diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp index 85b81c1deb23df..4ac9117fad9ae3 100644 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ b/be/src/io/fs/azure_obj_storage_client.cpp @@ -43,6 +43,7 @@ #include "common/exception.h" #include "common/logging.h" #include "common/status.h" +#include "cpp/obj_retry_strategy.h" #include "io/fs/obj_storage_client.h" #include "util/bvar_helper.h" #include "util/coding.h" @@ -74,7 +75,7 @@ auto s3_rate_limit(doris::S3RateLimitType op, Func callback) -> decltype(callbac if (!doris::config::enable_s3_rate_limiter) { return callback(); } - auto sleep_duration = doris::S3ClientFactory::instance().rate_limiter(op)->add(1); + auto sleep_duration = doris::apply_s3_rate_limit(op); if (sleep_duration < 0) { throw std::runtime_error("Azure exceeds request limit"); } @@ -124,6 +125,7 @@ ObjectStorageResponse do_azure_client_call(Func f, const ObjectStoragePathOption try { f(); } catch (Azure::Core::RequestFailedException& e) { + doris::record_object_request_failed(static_cast(e.StatusCode)); auto tls_debug_suffix = build_azure_tls_debug_suffix( fmt::format("{} {}", e.what(), e.Message), tls_debug_context); auto msg = fmt::format( @@ -186,6 +188,7 @@ struct AzureBatchDeleter { 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) { continue; } + doris::record_object_request_failed(static_cast(e.StatusCode)); auto msg = fmt::format( "Azure request failed because {}, error msg {}, http code {}, path msg " "{}{}", diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index 246ee5e480db5d..386c3e4192c3d6 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -32,6 +32,7 @@ #include "common/status.h" #include "core/custom_allocator.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -655,7 +656,9 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File std::function sync_buffer = nullptr; if (profile != nullptr) { const char* prefetch_buffered_reader = "PrefetchBufferedReader"; - ADD_TIMER(profile, prefetch_buffered_reader); + auto* total_time = + ADD_CHILD_TIMER(profile, prefetch_buffered_reader, + file_scan_profile::parent_or_root(profile, file_scan_profile::IO)); auto copy_time = ADD_CHILD_TIMER(profile, "CopyTime", prefetch_buffered_reader); auto read_time = ADD_CHILD_TIMER(profile, "ReadTime", prefetch_buffered_reader); auto prefetch_request_io = @@ -667,6 +670,7 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File auto request_bytes = ADD_CHILD_COUNTER(profile, "RequestBytes", TUnit::BYTES, prefetch_buffered_reader); sync_buffer = [=](PrefetchBuffer& buf) { + COUNTER_UPDATE(total_time, buf._statis.copy_time + buf._statis.read_time); COUNTER_UPDATE(copy_time, buf._statis.copy_time); COUNTER_UPDATE(read_time, buf._statis.read_time); COUNTER_UPDATE(prefetch_request_io, buf._statis.prefetch_request_io); @@ -924,7 +928,9 @@ RangeCacheFileReader::RangeCacheFileReader(RuntimeProfile* profile, io::FileRead if (_profile != nullptr) { const char* random_profile = "RangeCacheFileReader"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, random_profile, 1); _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, @@ -985,6 +991,7 @@ Status RangeCacheFileReader::read_at_impl(size_t offset, Slice result, size_t* b void RangeCacheFileReader::_collect_profile_before_close() { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _cache_statistics.request_time); COUNTER_UPDATE(_request_io, _cache_statistics.request_io); COUNTER_UPDATE(_request_bytes, _cache_statistics.request_bytes); COUNTER_UPDATE(_request_time, _cache_statistics.request_time); diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h index beaf3e87a8d3ce..9b0e66ebbd078b 100644 --- a/be/src/io/fs/buffered_reader.h +++ b/be/src/io/fs/buffered_reader.h @@ -38,6 +38,7 @@ #include "io/fs/path.h" #include "io/fs/s3_file_reader.h" #include "io/io_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/olap_define.h" #include "util/slice.h" @@ -179,6 +180,7 @@ class RangeCacheFileReader : public io::FileReader { bool _closed = false; RuntimeProfile::Counter* _request_io = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _request_bytes = nullptr; RuntimeProfile::Counter* _request_time = nullptr; RuntimeProfile::Counter* _read_to_cache_time = nullptr; @@ -301,7 +303,9 @@ class MergeRangeFileReader : public io::FileReader { if (_profile != nullptr) { const char* random_profile = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", random_profile, 1); _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", random_profile, 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, @@ -350,6 +354,7 @@ class MergeRangeFileReader : public io::FileReader { void _collect_profile_before_close() override { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -363,6 +368,7 @@ class MergeRangeFileReader : public io::FileReader { } private: + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _copy_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index d99db30f1c00fc..6e363636980d19 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -32,6 +32,7 @@ #include "cpp/sync_point.h" #include "io/fs/err_utils.h" #include "io/hdfs_util.h" +#include "runtime/file_scan_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" #include "service/backend_options.h" @@ -83,7 +84,9 @@ HdfsFileReader::HdfsFileReader(Path path, std::string fs_name, FileHandleCache:: if (_profile != nullptr && is_hdfs(_fs_name)) { #ifdef USE_HADOOP_HDFS const char* hdfs_profile_name = "HdfsIO"; - ADD_TIMER(_profile, hdfs_profile_name); + _total_read_time = + ADD_CHILD_TIMER(_profile, hdfs_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); _hdfs_profile.total_bytes_read = ADD_CHILD_COUNTER(_profile, "TotalBytesRead", TUnit::BYTES, hdfs_profile_name); _hdfs_profile.total_local_bytes_read = @@ -117,6 +120,7 @@ Status HdfsFileReader::close() { Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, const IOContext* io_ctx) { + SCOPED_TIMER(_total_read_time); auto st = do_read_at_impl(offset, result, bytes_read, io_ctx); if (!st.ok()) { _handle = nullptr; diff --git a/be/src/io/fs/hdfs_file_reader.h b/be/src/io/fs/hdfs_file_reader.h index 08f98bca29af0c..7bb73f30909452 100644 --- a/be/src/io/fs/hdfs_file_reader.h +++ b/be/src/io/fs/hdfs_file_reader.h @@ -88,6 +88,7 @@ class HdfsFileReader final : public FileReader { CachedHdfsFileHandle* _handle = nullptr; // owned by _cached_file_handle std::atomic _closed = false; RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* _total_read_time = nullptr; int64_t _mtime; #ifdef USE_HADOOP_HDFS HDFSProfile _hdfs_profile; diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 4eaa10f3311e06..af8dde36d2df50 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -37,6 +37,7 @@ #include "io/fs/err_utils.h" #include "io/fs/obj_storage_client.h" #include "io/fs/s3_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -214,7 +215,9 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea void S3FileReader::_collect_profile_before_close() { if (_profile != nullptr) { const char* s3_profile_name = "S3Profile"; - ADD_TIMER(_profile, s3_profile_name); + auto* total_time = + ADD_CHILD_TIMER(_profile, s3_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); RuntimeProfile::Counter* total_get_request_counter = ADD_CHILD_COUNTER(_profile, "TotalGetRequest", TUnit::UNIT, s3_profile_name); RuntimeProfile::Counter* too_many_request_err_counter = @@ -231,6 +234,7 @@ void S3FileReader::_collect_profile_before_close() { COUNTER_UPDATE(too_many_request_sleep_time, _s3_stats.too_many_request_sleep_time_ms); COUNTER_UPDATE(total_bytes_read, _s3_stats.total_bytes_read); COUNTER_UPDATE(total_get_request_time_ns, _s3_stats.total_get_request_time_ns); + COUNTER_UPDATE(total_time, _s3_stats.total_get_request_time_ns); } } diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index 1b8bf7b473c076..f9ed8e155ff59c 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -66,6 +66,7 @@ #include "common/logging.h" #include "common/status.h" +#include "cpp/obj_retry_strategy.h" #include "cpp/sync_point.h" #include "io/fs/err_utils.h" #include "io/fs/s3_common.h" @@ -82,7 +83,7 @@ auto s3_rate_limit(doris::S3RateLimitType op, Func callback) -> decltype(callbac if (!doris::config::enable_s3_rate_limiter) { return callback(); } - auto sleep_duration = doris::S3ClientFactory::instance().rate_limiter(op)->add(1); + auto sleep_duration = doris::apply_s3_rate_limit(op); if (sleep_duration < 0) { return T(s3_error_factory()); } @@ -98,6 +99,10 @@ template auto s3_put_rate_limit(Func callback) -> decltype(callback()) { return s3_rate_limit(doris::S3RateLimitType::PUT, std::move(callback)); } + +void record_s3_request_failed(const Aws::S3::S3Error& error) { + doris::record_object_request_failed(static_cast(error.GetResponseCode())); +} } // namespace namespace Aws::S3::Model { @@ -140,6 +145,7 @@ ObjectStorageUploadResponse S3ObjStorageClient::create_multipart_upload( << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); auto st = s3fs_error(outcome.GetError(), fmt::format("failed to CreateMultipartUpload: {} ", opts.path.native())); LOG(WARNING) << st << " request_id=" << request_id; @@ -177,6 +183,7 @@ ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOpti : outcome.GetError().GetRequestId(); if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); auto st = s3fs_error(outcome.GetError(), fmt::format("failed to put object: {}", opts.path.native())); LOG(WARNING) << st << ", request_id=" << request_id; @@ -222,6 +229,7 @@ ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStorageP TEST_SYNC_POINT_CALLBACK("S3FileWriter::_upload_one_part", &outcome); if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); auto st = Status::IOError( "failed to UploadPart bucket={}, key={}, part_num={}, upload_id={}, message={}, " "exception_name={}, response_code={}, request_id={}", @@ -275,6 +283,7 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( : outcome.GetError().GetRequestId(); if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); auto st = s3fs_error(outcome.GetError(), fmt::format("failed to CompleteMultipartUpload: {}, upload_id={}", opts.path.native(), *opts.upload_id)); @@ -305,6 +314,7 @@ ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePat } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { return {.resp = {convert_to_obj_response(Status::Error(""))}}; } else { + record_s3_request_failed(outcome.GetError()); return {.resp = {convert_to_obj_response( s3fs_error(outcome.GetError(), fmt::format("failed to check exists {}", opts.key))), @@ -324,6 +334,7 @@ ObjectStorageResponse S3ObjStorageClient::get_object(const ObjectStoragePathOpti SCOPED_BVAR_LATENCY(s3_bvar::s3_get_latency); auto outcome = s3_get_rate_limit([&]() { return _client->GetObject(request); }); if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); return {convert_to_obj_response(s3fs_error( outcome.GetError(), fmt::format("failed to read from {}", opts.key))), static_cast(outcome.GetError().GetResponseCode()), @@ -362,6 +373,7 @@ ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOp return ObjectStorageResponse::OK(); } + record_s3_request_failed(outcome.GetError()); return {convert_to_obj_response(s3fs_error( outcome.GetError(), fmt::format("failed to list {}", opts.prefix))), static_cast(outcome.GetError().GetResponseCode()), @@ -406,6 +418,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePath auto delete_outcome = s3_put_rate_limit([&]() { return _client->DeleteObjects(delete_request); }); if (!delete_outcome.IsSuccess()) { + record_s3_request_failed(delete_outcome.GetError()); return {convert_to_obj_response( s3fs_error(delete_outcome.GetError(), fmt::format("failed to delete dir {}", opts.key))), @@ -433,6 +446,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_object(const ObjectStoragePathO outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { return ObjectStorageResponse::OK(); } + record_s3_request_failed(outcome.GetError()); return {convert_to_obj_response(s3fs_error(outcome.GetError(), fmt::format("failed to delete file {}", opts.key))), static_cast(outcome.GetError().GetResponseCode()), @@ -453,6 +467,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( outcome = s3_get_rate_limit([&]() { return _client->ListObjectsV2(request); }); } if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); return {convert_to_obj_response(s3fs_error( outcome.GetError(), fmt::format("failed to list objects when delete dir {}", opts.prefix))), @@ -473,6 +488,7 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( auto delete_outcome = s3_put_rate_limit([&]() { return _client->DeleteObjects(delete_request); }); if (!delete_outcome.IsSuccess()) { + record_s3_request_failed(delete_outcome.GetError()); return {convert_to_obj_response( s3fs_error(delete_outcome.GetError(), fmt::format("failed to delete dir {}", opts.key))), @@ -502,4 +518,4 @@ std::string S3ObjStorageClient::generate_presigned_url(const ObjectStoragePathOp expiration_secs); } -} // namespace doris::io \ No newline at end of file +} // namespace doris::io diff --git a/be/src/io/fs/stream_sink_file_writer.cpp b/be/src/io/fs/stream_sink_file_writer.cpp index 1316cd71fac4e9..e4a4a4d932e5fc 100644 --- a/be/src/io/fs/stream_sink_file_writer.cpp +++ b/be/src/io/fs/stream_sink_file_writer.cpp @@ -19,6 +19,8 @@ #include +#include + #include "exec/sink/load_stream_stub.h" #include "storage/olap_common.h" #include "storage/rowset/beta_rowset_writer.h" @@ -27,6 +29,28 @@ namespace doris::io { +std::unordered_set StreamSinkFileWriter::_get_fault_injection_failed_dst_ids() const { + size_t failed_replica_num = 0; + DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", + { failed_replica_num = 1; }); + DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", + { failed_replica_num = 2; }); + DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", + { failed_replica_num = _streams.size(); }); + if (failed_replica_num == 0) { + return {}; + } + + std::vector dst_ids; + dst_ids.reserve(_streams.size()); + for (const auto& stream : _streams) { + dst_ids.push_back(stream->dst_id()); + } + std::sort(dst_ids.begin(), dst_ids.end()); + dst_ids.resize(std::min(failed_replica_num, dst_ids.size())); + return std::unordered_set(dst_ids.begin(), dst_ids.end()); +} + void StreamSinkFileWriter::init(PUniqueId load_id, int64_t partition_id, int64_t index_id, int64_t tablet_id, int32_t segment_id, FileType file_type) { VLOG_DEBUG << "init stream writer, load id(" << UniqueId(load_id).to_string() @@ -52,24 +76,16 @@ Status StreamSinkFileWriter::appendv(const Slice* data, size_t data_cnt) { << ", data_length: " << bytes_req << "file_type" << _file_type; std::span slices {data, data_cnt}; - size_t fault_injection_skipped_streams = 0; + auto fault_injection_failed_dst_ids = _get_fault_injection_failed_dst_ids(); bool ok = false; Status st; for (auto& stream : _streams) { - DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", { - if (fault_injection_skipped_streams < 1) { - fault_injection_skipped_streams++; - continue; - } - }); - DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", { - if (fault_injection_skipped_streams < 2) { - fault_injection_skipped_streams++; - continue; - } - }); - DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", - { continue; }); + if (fault_injection_failed_dst_ids.contains(stream->dst_id())) { + LOG(INFO) << "fault injection skips segment data to backend " << stream->dst_id() + << ", load_id: " << print_id(_load_id) << ", index_id: " << _index_id + << ", tablet_id: " << _tablet_id << ", segment_id: " << _segment_id; + continue; + } st = stream->append_data(_partition_id, _index_id, _tablet_id, _segment_id, _bytes_appended, slices, false, _file_type); ok = ok || st.ok(); @@ -123,23 +139,15 @@ Status StreamSinkFileWriter::_finalize() { VLOG_DEBUG << "writer finalize, load_id: " << print_id(_load_id) << ", index_id: " << _index_id << ", tablet_id: " << _tablet_id << ", segment_id: " << _segment_id; // TODO(zhengyu): update get_inverted_index_file_size into stat - size_t fault_injection_skipped_streams = 0; + auto fault_injection_failed_dst_ids = _get_fault_injection_failed_dst_ids(); bool ok = false; for (auto& stream : _streams) { - DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", { - if (fault_injection_skipped_streams < 1) { - fault_injection_skipped_streams++; - continue; - } - }); - DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", { - if (fault_injection_skipped_streams < 2) { - fault_injection_skipped_streams++; - continue; - } - }); - DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", - { continue; }); + if (fault_injection_failed_dst_ids.contains(stream->dst_id())) { + LOG(INFO) << "fault injection skips segment eos to backend " << stream->dst_id() + << ", load_id: " << print_id(_load_id) << ", index_id: " << _index_id + << ", tablet_id: " << _tablet_id << ", segment_id: " << _segment_id; + continue; + } auto st = stream->append_data(_partition_id, _index_id, _tablet_id, _segment_id, _bytes_appended, {}, true, _file_type); ok = ok || st.ok(); diff --git a/be/src/io/fs/stream_sink_file_writer.h b/be/src/io/fs/stream_sink_file_writer.h index f092319e7fa24c..74f7d4e091ca32 100644 --- a/be/src/io/fs/stream_sink_file_writer.h +++ b/be/src/io/fs/stream_sink_file_writer.h @@ -21,6 +21,7 @@ #include #include +#include #include "io/fs/file_writer.h" #include "util/uid_util.h" @@ -57,6 +58,7 @@ class StreamSinkFileWriter final : public FileWriter { private: Status _finalize(); + std::unordered_set _get_fault_injection_failed_dst_ids() const; std::vector> _streams; PUniqueId _load_id; diff --git a/be/src/load/group_commit/group_commit_mgr.cpp b/be/src/load/group_commit/group_commit_mgr.cpp index 930c518ae9afb5..b0dd97ed1303d2 100644 --- a/be/src/load/group_commit/group_commit_mgr.cpp +++ b/be/src/load/group_commit/group_commit_mgr.cpp @@ -31,6 +31,7 @@ #include "runtime/fragment_mgr.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" +#include "service/backend_options.h" #include "util/client_cache.h" #include "util/debug_points.h" #include "util/thrift_rpc_helper.h" @@ -267,16 +268,19 @@ void LoadBlockQueue::_cancel_without_lock(const Status& st) { Status GroupCommitTable::get_first_block_load_queue( int64_t table_id, int64_t base_schema_version, int64_t index_size, const UniqueId& load_id, std::shared_ptr& load_block_queue, int be_exe_version, - std::shared_ptr create_plan_dep, std::shared_ptr put_block_dep) { + TGroupCommitMode::type group_commit_mode, std::shared_ptr create_plan_dep, + std::shared_ptr put_block_dep) { DCHECK(table_id == _table_id); std::unique_lock l(_lock); - auto try_to_get_matched_queue = [&]() -> Status { + auto try_to_get_matched_queue = [&](bool& need_create_plan) -> Status { + need_create_plan = false; for (const auto& [_, inner_block_queue] : _load_block_queues) { if (inner_block_queue->contain_load_id(load_id)) { load_block_queue = inner_block_queue; return Status::OK(); } } + RETURN_IF_ERROR(_check_wal_backlog(group_commit_mode)); for (const auto& [_, inner_block_queue] : _load_block_queues) { if (!inner_block_queue->need_commit()) { if (base_schema_version == inner_block_queue->schema_version && @@ -292,11 +296,13 @@ Status GroupCommitTable::get_first_block_load_queue( } } } - return Status::InternalError("can not get a block queue for table_id: " + - std::to_string(_table_id) + _create_plan_failed_reason); + need_create_plan = true; + return Status::OK(); }; - if (try_to_get_matched_queue().ok()) { + bool need_create_plan = false; + RETURN_IF_ERROR(try_to_get_matched_queue(need_create_plan)); + if (!need_create_plan) { return Status::OK(); } create_plan_dep->block(); @@ -307,7 +313,31 @@ Status GroupCommitTable::get_first_block_load_queue( _create_plan_deps.emplace(load_id, std::make_tuple(create_plan_dep, put_block_dep, base_schema_version, index_size)); [[maybe_unused]] auto submit_st = _submit_create_group_commit_load(); - return try_to_get_matched_queue(); + RETURN_IF_ERROR(try_to_get_matched_queue(need_create_plan)); + if (need_create_plan) { + return Status::InternalError("can not get a block queue for table_id: " + + std::to_string(_table_id) + _create_plan_failed_reason); + } + return Status::OK(); +} + +Status GroupCommitTable::_check_wal_backlog(TGroupCommitMode::type group_commit_mode) { + int32_t max_wal_num = config::group_commit_max_wal_num_per_table; + if (group_commit_mode != TGroupCommitMode::ASYNC_MODE || max_wal_num <= 0) { + return Status::OK(); + } + size_t wal_num = _exec_env->wal_mgr()->get_wal_queue_size(_table_id); + if (wal_num < static_cast(max_wal_num)) { + return Status::OK(); + } + std::string failed_reason = _exec_env->wal_mgr()->get_last_replay_wal_failed_reason(_table_id); + if (failed_reason.empty()) { + return Status::OK(); + } + return Status::Error( + "Too many group commit async WALs for table {} on be host {}. wal num={}, limit={}, " + "last replay wal failed reason: {}", + _table_id, BackendOptions::get_localhost(), wal_num, max_wal_num, failed_reason); } Status GroupCommitTable::submit_create_group_commit_load() { @@ -840,13 +870,11 @@ void GroupCommitMgr::_create_plan_worker() { } } -Status GroupCommitMgr::get_first_block_load_queue(int64_t db_id, int64_t table_id, - int64_t base_schema_version, int64_t index_size, - const UniqueId& load_id, - std::shared_ptr& load_block_queue, - int be_exe_version, - std::shared_ptr create_plan_dep, - std::shared_ptr put_block_dep) { +Status GroupCommitMgr::get_first_block_load_queue( + int64_t db_id, int64_t table_id, int64_t base_schema_version, int64_t index_size, + const UniqueId& load_id, std::shared_ptr& load_block_queue, + int be_exe_version, TGroupCommitMode::type group_commit_mode, + std::shared_ptr create_plan_dep, std::shared_ptr put_block_dep) { std::shared_ptr group_commit_table; { std::lock_guard wlock(_lock); @@ -859,7 +887,7 @@ Status GroupCommitMgr::get_first_block_load_queue(int64_t db_id, int64_t table_i } RETURN_IF_ERROR(group_commit_table->get_first_block_load_queue( table_id, base_schema_version, index_size, load_id, load_block_queue, be_exe_version, - create_plan_dep, put_block_dep)); + group_commit_mode, create_plan_dep, put_block_dep)); return Status::OK(); } diff --git a/be/src/load/group_commit/group_commit_mgr.h b/be/src/load/group_commit/group_commit_mgr.h index 5ab3831e4f5cc4..895715afdeedc1 100644 --- a/be/src/load/group_commit/group_commit_mgr.h +++ b/be/src/load/group_commit/group_commit_mgr.h @@ -166,7 +166,7 @@ class GroupCommitTable { Status get_first_block_load_queue(int64_t table_id, int64_t base_schema_version, int64_t index_size, const UniqueId& load_id, std::shared_ptr& load_block_queue, - int be_exe_version, + int be_exe_version, TGroupCommitMode::type group_commit_mode, std::shared_ptr create_plan_dep, std::shared_ptr put_block_dep); Status get_load_block_queue(const TUniqueId& instance_id, @@ -177,6 +177,7 @@ class GroupCommitTable { private: Status _submit_create_group_commit_load(); + Status _check_wal_backlog(TGroupCommitMode::type group_commit_mode); Status _create_group_commit_load(int be_exe_version, const std::shared_ptr& mem_tracker, std::shared_ptr& created_load_block_queue); @@ -222,7 +223,7 @@ class GroupCommitMgr { Status get_first_block_load_queue(int64_t db_id, int64_t table_id, int64_t base_schema_version, int64_t index_size, const UniqueId& load_id, std::shared_ptr& load_block_queue, - int be_exe_version, + int be_exe_version, TGroupCommitMode::type group_commit_mode, std::shared_ptr create_plan_dep, std::shared_ptr put_block_dep); void remove_load_id(int64_t table_id, const UniqueId& load_id); diff --git a/be/src/load/group_commit/wal/wal_manager.cpp b/be/src/load/group_commit/wal/wal_manager.cpp index 06d009404f7efa..e7962cc3396ce1 100644 --- a/be/src/load/group_commit/wal/wal_manager.cpp +++ b/be/src/load/group_commit/wal/wal_manager.cpp @@ -194,7 +194,7 @@ void WalManager::erase_wal_queue(int64_t table_id, int64_t wal_id) { } size_t WalManager::get_wal_queue_size(int64_t table_id) { - std::lock_guard wrlock(_wal_queue_lock); + std::shared_lock rdlock(_wal_queue_lock); size_t count = 0; if (table_id > 0) { auto it = _wal_queues.find(table_id); @@ -206,7 +206,7 @@ size_t WalManager::get_wal_queue_size(int64_t table_id) { } else { // table_id is -1 meaning get all table wal size size_t max_count_per_table = 0; - for (auto& [_, table_wals] : _wal_queues) { + for (const auto& [_, table_wals] : _wal_queues) { size_t table_wal_count = table_wals.size(); count += table_wal_count; if (table_wal_count > max_count_per_table) { @@ -218,6 +218,15 @@ size_t WalManager::get_wal_queue_size(int64_t table_id) { return count; } +std::string WalManager::get_last_replay_wal_failed_reason(int64_t table_id) { + std::shared_lock rdlock(_table_lock); + auto it = _table_map.find(table_id); + if (it != _table_map.end()) { + return it->second->get_last_replay_wal_failed_reason(); + } + return ""; +} + Status WalManager::create_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id, const std::string& label, std::string& base_path, uint32_t wal_version) { diff --git a/be/src/load/group_commit/wal/wal_manager.h b/be/src/load/group_commit/wal/wal_manager.h index 4157cc11d19f4e..48985e0cba8074 100644 --- a/be/src/load/group_commit/wal/wal_manager.h +++ b/be/src/load/group_commit/wal/wal_manager.h @@ -80,6 +80,7 @@ class WalManager { void add_wal_queue(int64_t table_id, int64_t wal_id); void erase_wal_queue(int64_t table_id, int64_t wal_id); size_t get_wal_queue_size(int64_t table_id); + std::string get_last_replay_wal_failed_reason(int64_t table_id); // filename format:a_b_c_group_commit_xxx // a:version // b:be id diff --git a/be/src/load/group_commit/wal/wal_table.cpp b/be/src/load/group_commit/wal/wal_table.cpp index 5dfd54354c3174..060df525050807 100644 --- a/be/src/load/group_commit/wal/wal_table.cpp +++ b/be/src/load/group_commit/wal/wal_table.cpp @@ -113,6 +113,12 @@ Status WalTable::_relay_wal_one_by_one() { doris::wal_fail << 1; LOG(WARNING) << "failed to replay wal=" << wal_info->get_wal_path() << ", st=" << st.to_string(); + { + std::lock_guard lock(_replay_wal_lock); + _last_replay_wal_failed_reason = + "failed to replay wal=" + wal_info->get_wal_path() + + ", st=" + st.to_string().substr(0, 100); + } need_retry_wals.push_back(wal_info); } } @@ -122,6 +128,9 @@ Status WalTable::_relay_wal_one_by_one() { for (auto retry_wal_info : need_retry_wals) { _replay_wal_map.emplace(retry_wal_info->get_wal_path(), retry_wal_info); } + if (_replay_wal_map.empty()) { + _last_replay_wal_failed_reason.clear(); + } } return Status::OK(); } @@ -308,6 +317,11 @@ size_t WalTable::size() { return _replay_wal_map.size() + _replaying_queue.size(); } +std::string WalTable::get_last_replay_wal_failed_reason() const { + std::lock_guard lock(_replay_wal_lock); + return _last_replay_wal_failed_reason; +} + Status WalTable::_get_column_info(int64_t db_id, int64_t tb_id, std::map& column_info_map) { TGetColumnInfoRequest request; diff --git a/be/src/load/group_commit/wal/wal_table.h b/be/src/load/group_commit/wal/wal_table.h index 89223c65668e0d..57a0acfb6cf4ec 100644 --- a/be/src/load/group_commit/wal/wal_table.h +++ b/be/src/load/group_commit/wal/wal_table.h @@ -40,6 +40,7 @@ class WalTable { Status replay_wals(); size_t size(); void stop(); + std::string get_last_replay_wal_failed_reason() const; private: void _pick_relay_wals(); @@ -67,5 +68,6 @@ class WalTable { // key is wal_path std::map> _replay_wal_map; std::list> _replaying_queue; + std::string _last_replay_wal_failed_reason; }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/load/stream_load/stream_load_executor.cpp b/be/src/load/stream_load/stream_load_executor.cpp index 478ed22755df9e..9b307c932d4776 100644 --- a/be/src/load/stream_load/stream_load_executor.cpp +++ b/be/src/load/stream_load/stream_load_executor.cpp @@ -110,6 +110,9 @@ Status StreamLoadExecutor::execute_plan_fragment( *status = Status::DataQualityError("too many filtered rows"); } } + if (ctx->load_type == TLoadType::ROUTINE_LOAD || !status->ok()) { + ctx->first_error_msg = state->get_first_error_msg(); + } if (status->ok()) { DorisMetrics::instance()->stream_receive_bytes_total->increment(ctx->receive_bytes); @@ -118,7 +121,6 @@ Status StreamLoadExecutor::execute_plan_fragment( LOG(WARNING) << "fragment execute failed" << ", err_msg=" << status->to_string() << ", " << ctx->brief(); ctx->number_loaded_rows = 0; - ctx->first_error_msg = state->get_first_error_msg(); // cancel body_sink, make sender known it if (ctx->body_sink != nullptr) { ctx->body_sink->cancel(status->to_string()); @@ -428,6 +430,9 @@ bool StreamLoadExecutor::collect_load_stat(StreamLoadContext* ctx, TTxnCommitAtt rl_attach.__set_receivedBytes(ctx->receive_bytes); rl_attach.__set_loadedBytes(ctx->loaded_bytes); rl_attach.__set_loadCostMs(ctx->load_cost_millis); + if (!ctx->first_error_msg.empty()) { + rl_attach.__set_firstErrorMsg(ctx->first_error_msg); + } attach->rlTaskTxnCommitAttachment = rl_attach; attach->__isset.rlTaskTxnCommitAttachment = true; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index c010c1deeba272..97d068f0754f2e 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -517,7 +517,6 @@ void ExecEnv::init_file_cache_factory(std::vector& cache_paths config::file_cache_each_block_size, config::s3_write_buffer_size); exit(-1); } - std::unordered_set cache_path_set; Status rest = doris::parse_conf_cache_paths(doris::config::file_cache_path, cache_paths); if (!rest) { throw Exception( @@ -525,23 +524,16 @@ void ExecEnv::init_file_cache_factory(std::vector& cache_paths doris::config::file_cache_path, rest.msg())); } - doris::Status cache_status; - for (auto& cache_path : cache_paths) { - if (cache_path_set.find(cache_path.path) != cache_path_set.end()) { - LOG(WARNING) << fmt::format("cache path {} is duplicate", cache_path.path); - continue; - } - - cache_status = doris::io::FileCacheFactory::instance()->create_file_cache( - cache_path.path, cache_path.init_settings()); - if (!cache_status.ok()) { - if (!doris::config::ignore_broken_disk) { - throw Exception( - Status::FatalError("failed to init file cache, err: {}", cache_status)); - } - LOG(WARNING) << "failed to init file cache, err: " << cache_status; - } - cache_path_set.emplace(cache_path.path); + auto cache_status = doris::io::FileCacheFactory::instance()->create_file_caches( + cache_paths, [](const std::string&, const Status& status) { + if (!doris::config::ignore_broken_disk) { + return false; + } + LOG(WARNING) << "failed to init file cache, err: " << status; + return true; + }); + if (!cache_status.ok()) { + throw Exception(Status::FatalError("failed to init file cache, err: {}", cache_status)); } } diff --git a/be/src/runtime/file_scan_profile.h b/be/src/runtime/file_scan_profile.h new file mode 100644 index 00000000000000..4ec71c4f4820d4 --- /dev/null +++ b/be/src/runtime/file_scan_profile.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "runtime/runtime_profile.h" + +namespace doris::file_scan_profile { + +inline constexpr const char* SCANNER = "FileScannerV2"; +inline constexpr const char* TABLE_READER = "TableReader"; +inline constexpr const char* FILE_READER = "FileReader"; +inline constexpr const char* IO = "IO"; + +struct Hierarchy { + RuntimeProfile::Counter* scanner = nullptr; + RuntimeProfile::Counter* table_reader = nullptr; + RuntimeProfile::Counter* file_reader = nullptr; + RuntimeProfile::Counter* io = nullptr; +}; + +inline Hierarchy ensure_hierarchy(RuntimeProfile* profile) { + if (profile == nullptr) { + return {}; + } + // RuntimeProfile stores one flat counter namespace and a separate parent map. Create every + // layer in order so later format and IO profiles cannot accidentally become scanner siblings. + auto* scanner = ADD_TIMER_WITH_LEVEL(profile, SCANNER, 1); + auto* table_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, TABLE_READER, SCANNER, 1); + auto* file_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, FILE_READER, TABLE_READER, 1); + auto* io = ADD_CHILD_TIMER_WITH_LEVEL(profile, IO, FILE_READER, 1); + return {scanner, table_reader, file_reader, io}; +} + +inline const char* parent_or_root(RuntimeProfile* profile, const char* preferred_parent) { + return profile != nullptr && profile->get_counter(preferred_parent) != nullptr + ? preferred_parent + : RuntimeProfile::ROOT_COUNTER.c_str(); +} + +} // namespace doris::file_scan_profile diff --git a/be/src/runtime/query_cache/query_cache.cpp b/be/src/runtime/query_cache/query_cache.cpp index 28610d44808686..d7e042e0301a24 100644 --- a/be/src/runtime/query_cache/query_cache.cpp +++ b/be/src/runtime/query_cache/query_cache.cpp @@ -17,7 +17,17 @@ #include "runtime/query_cache/query_cache.h" +#include + +#include "cloud/config.h" #include "common/logging.h" +#include "common/metrics/doris_metrics.h" +#include "cpp/sync_point.h" +#include "storage/olap_common.h" +#include "storage/rowset/rowset.h" +#include "storage/rowset/rowset_reader.h" +#include "storage/tablet/base_tablet.h" +#include "storage/tablet/tablet_meta.h" namespace doris { @@ -39,8 +49,27 @@ int64_t QueryCacheHandle::get_cache_version() { return ((QueryCache::CacheValue*)(result_ptr))->version; } +int64_t QueryCacheHandle::get_cache_delta_count() { + DCHECK(_handle); + auto result_ptr = reinterpret_cast(_handle)->value; + return ((QueryCache::CacheValue*)(result_ptr))->delta_count; +} + +int64_t QueryCacheHandle::get_cache_total_bytes() { + DCHECK(_handle); + auto result_ptr = reinterpret_cast(_handle)->value; + return ((QueryCache::CacheValue*)(result_ptr))->total_bytes; +} + +int64_t QueryCacheHandle::get_cache_total_rows() { + DCHECK(_handle); + auto result_ptr = reinterpret_cast(_handle)->value; + return ((QueryCache::CacheValue*)(result_ptr))->total_rows; +} + void QueryCache::insert(const CacheKey& key, int64_t version, CacheResult& res, - const std::vector& slot_orders, int64_t cache_size) { + const std::vector& slot_orders, int64_t cache_size, + int64_t delta_count) { SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->query_cache_mem_tracker()); CacheResult cache_result; for (auto& block_data : res) { @@ -50,11 +79,12 @@ void QueryCache::insert(const CacheKey& key, int64_t version, CacheResult& res, auto st = mutable_block.merge(*block_data); DORIS_CHECK(st.ok()); } - auto cache_value_ptr = - std::make_unique(version, std::move(cache_result), slot_orders); + auto cache_value_ptr = std::make_unique( + version, std::move(cache_result), slot_orders, delta_count, cache_size); QueryCacheHandle(this, LRUCachePolicy::insert(key, (void*)cache_value_ptr.release(), cache_size, cache_size, CachePriority::NORMAL)); + DorisMetrics::instance()->query_cache_write_back_total->increment(1); } bool QueryCache::lookup(const CacheKey& key, int64_t version, doris::QueryCacheHandle* handle) { @@ -70,4 +100,375 @@ bool QueryCache::lookup(const CacheKey& key, int64_t version, doris::QueryCacheH return false; } +bool QueryCache::lookup_any_version(const CacheKey& key, QueryCacheHandle* handle) { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->query_cache_mem_tracker()); + auto* lru_handle = LRUCachePolicy::lookup(key); + if (lru_handle) { + *handle = QueryCacheHandle(this, lru_handle); + return true; + } + return false; +} + +QueryCacheInstanceDecision::~QueryCacheInstanceDecision() = default; + +std::unique_ptr QueryCacheInstanceDecision::take_delta_read_source( + int64_t tablet_id) { + std::lock_guard lock(_take_lock); + auto it = _delta_read_sources.find(tablet_id); + if (it == _delta_read_sources.end()) { + return nullptr; + } + auto res = std::move(it->second); + _delta_read_sources.erase(it); + return res; +} + +std::shared_ptr QueryCacheRuntime::get_or_make_decision( + const std::vector& scan_ranges) { + std::string cache_key; + int64_t version = 0; + Status st = QueryCache::build_cache_key(scan_ranges, _param, &cache_key, &version); + if (!st.ok()) { + // No reliable cache key for this instance (e.g. FE could not align the + // instance to a single partition so tablets carry different versions). + // Degrade to an uncached scan instead of failing the query: both the + // scan operator and the cache source operator observe the shared MISS + // decision with key_valid=false, so nothing is looked up and nothing + // is written back. This is an expected plan shape, so log only once + // per fragment instead of twice per instance. + std::lock_guard lock(_lock); + if (_invalid_decision == nullptr) { + LOG(WARNING) << "query cache degrades to uncached scan, node_id=" << _param.node_id + << ", reason: " << st.to_string(); + _invalid_decision = std::make_shared(); + } + return _invalid_decision; + } + + { + std::lock_guard lock(_lock); + auto it = _decisions.find(cache_key); + if (it != _decisions.end()) { + return it->second; + } + } + + // Build the candidate outside the lock: a stale-entry decision walks + // tablet metadata under the header lock and, on merge-on-write tables, + // scans the delete bitmap, so its cost grows with the tablets per + // instance and the bitmap size. This runtime is fragment-wide, so a + // single lock-scoped build would serialize even unrelated instances' + // keys behind one slow decision. Racing callers of the same instance + // build duplicate candidates; the loser adopts the winner's decision + // below and its own candidate releases with the shared_ptr (the entry + // pin and any captured delta read sources go with it), which only costs + // a redundant metadata pass. The metrics are settled once, by the + // winner, after publication. + auto decision = std::make_shared(); + decision->key_valid = true; + decision->cache_key = cache_key; + decision->current_version = version; + _make_decision(scan_ranges, decision.get()); + TEST_SYNC_POINT("QueryCacheRuntime::get_or_make_decision.before_publish"); + + std::lock_guard lock(_lock); + auto [it, inserted] = _decisions.emplace(std::move(cache_key), decision); + if (inserted) { + if (decision->mode == QueryCacheInstanceDecision::Mode::INCREMENTAL) { + DorisMetrics::instance()->query_cache_stale_hit_total->increment(1); + } else if (decision->mode == QueryCacheInstanceDecision::Mode::MISS && + !decision->incremental_fallback_reason.empty()) { + // Only count real fallbacks: an empty reason means incremental + // merge was not enabled for this query in the first place. + DorisMetrics::instance()->query_cache_incremental_fallback_total->increment(1); + } + } + return it->second; +} + +void QueryCacheRuntime::_make_decision(const std::vector& scan_ranges, + QueryCacheInstanceDecision* decision) { + decision->mode = QueryCacheInstanceDecision::Mode::MISS; + if (_binlog_scan) { + // A row-binlog scan reads a different data stream under the same plan + // digest: it must neither serve cached blocks nor write its output + // back (that would poison the cache for normal queries). + decision->key_valid = false; + return; + } + if (_param.force_refresh_query_cache) { + return; + } + + QueryCacheHandle handle; + if (!_cache->lookup_any_version(decision->cache_key, &handle)) { + return; + } + // Pin the entry: as long as this decision object lives, the entry cannot be + // evicted, so every operator consuming this decision sees the same blocks. + decision->handle = std::move(handle); + + if (decision->handle.get_cache_version() == decision->current_version) { + decision->mode = QueryCacheInstanceDecision::Mode::HIT; + decision->cached_delta_count = decision->handle.get_cache_delta_count(); + return; + } + + if (_try_prepare_incremental(scan_ranges, decision)) { + decision->mode = QueryCacheInstanceDecision::Mode::INCREMENTAL; + return; + } + + // Stale entry not reusable: drop the pin and fall back to a full scan. + // The stale-hit/fallback metrics are settled by the caller once the + // winning candidate is published, so a losing racer cannot double-count. + decision->_delta_read_sources.clear(); + decision->handle = QueryCacheHandle(); + decision->mode = QueryCacheInstanceDecision::Mode::MISS; +} + +bool QueryCacheRuntime::_try_prepare_incremental(const std::vector& scan_ranges, + QueryCacheInstanceDecision* decision) { + if (!(_param.__isset.allow_incremental && _param.allow_incremental)) { + // Not a fallback: incremental merge is simply not enabled for this + // query, so leave incremental_fallback_reason empty. + return false; + } + // Cloud tablets capture rowsets through a different (partly asynchronous) + // path; incremental merge only supports local storage for now. + if (config::is_cloud_mode()) { + decision->incremental_fallback_reason = "cloud mode"; + return false; + } + + int64_t cached_version = decision->handle.get_cache_version(); + if (cached_version >= decision->current_version) { + // The entry is newer than what this replica is asked to read (e.g. the + // entry was filled from another replica with a newer visible version). + // A full scan of the requested version is the only safe answer. + decision->incremental_fallback_reason = "cached entry is newer"; + return false; + } + int64_t cached_delta_count = decision->handle.get_cache_delta_count(); + if (cached_delta_count >= config::query_cache_max_incremental_merge_count) { + // Force a full recompute to compact the entry: every incremental merge + // appends the delta blocks to the entry, so both the entry and the + // upstream merge get more fragmented as deltas accumulate. + decision->incremental_fallback_reason = "delta count reached compaction threshold"; + return false; + } + + for (const auto& scan_range : scan_ranges) { + int64_t tablet_id = scan_range.scan_range.palo_scan_range.tablet_id; + if (!_capture_tablet_delta(tablet_id, cached_version, decision)) { + return false; + } + } + + // If the cached entry alone already exceeds the entry limits, the merged + // entry (which can only be larger) could never be written back. Still scan + // only the delta, but tell the cache source upfront so it does not clone + // blocks for a write back that would be discarded anyway. Such an entry + // stays stale until compaction merges its delta away (then the capture + // above fails and a full recompute takes over). + if (decision->handle.get_cache_total_bytes() > _param.entry_max_bytes || + decision->handle.get_cache_total_rows() > _param.entry_max_rows) { + decision->write_back_feasible = false; + } + + decision->cached_version = cached_version; + decision->cached_delta_count = cached_delta_count; + return true; +} + +bool QueryCacheRuntime::_capture_tablet_delta(int64_t tablet_id, int64_t cached_version, + QueryCacheInstanceDecision* decision) { + auto tablet_res = ExecEnv::get_tablet(tablet_id); + if (!tablet_res) { + decision->incremental_fallback_reason = "tablet not found"; + return false; + } + BaseTabletSPtr tablet = std::move(tablet_res.value()); + // Defensive re-check of what FE promised with allow_incremental: + // "cached snapshot + delta rowsets == new snapshot" holds unconditionally + // for append-only DUP_KEYS data, and for merge-on-write UNIQUE data as + // long as the delta did not rewrite any row that predates the cached + // version (verified below once the delta rowsets are known). It never + // holds for merge-on-read UNIQUE data (duplicates are resolved by merging + // across rowsets at read time, so a delta-only scan cannot stand alone) + // nor for AGG tables (rows merge in the storage layer likewise). + const bool merge_on_write = tablet->keys_type() == KeysType::UNIQUE_KEYS && + tablet->enable_unique_key_merge_on_write(); + if (tablet->keys_type() != KeysType::DUP_KEYS && !merge_on_write) { + decision->incremental_fallback_reason = "keys type not append-only"; + return false; + } + + // quiet: on a version-graph miss (the delta merged away by compaction, an + // expected per-query situation) this option makes the capture API neither + // log nor error; it returns whatever PREFIX of the path it walked before + // the miss (empty when the window start itself is gone), so emptiness AND + // window coverage are both checked below as the failure signals. + auto source_res = tablet->capture_read_source({cached_version + 1, decision->current_version}, + {.quiet = true}); + if (!source_res) { + // Non-pathfinding failures (e.g. a rowset object missing for a found + // path) still surface as errors even under quiet. + decision->incremental_fallback_reason = "delta versions not capturable"; + return false; + } + auto read_source = std::make_unique(std::move(source_res.value())); + if (read_source->rs_splits.empty()) { + // A non-empty version window always lies on a consistent path with at + // least one rowset; an empty capture therefore means the delta + // versions are gone, never "no new data". Treating it as INCREMENTAL + // would silently drop the delta rows from the result. + decision->incremental_fallback_reason = "delta versions not capturable"; + return false; + } + // A quiet capture broken mid-window (e.g. a replica whose local view ends + // short of current_version) yields a partial prefix, and treating it as + // INCREMENTAL would silently drop the tail rows and poison the entry on + // write-back. The path is contiguous by construction, so checking both + // endpoints pins the whole (cached_version, current_version] window. + if (read_source->rs_splits.front().rs_reader->rowset()->start_version() != cached_version + 1 || + read_source->rs_splits.back().rs_reader->rowset()->end_version() != + decision->current_version) { + decision->incremental_fallback_reason = "delta versions not capturable"; + return false; + } + read_source->fill_delete_predicates(); + if (!read_source->delete_predicates.empty()) { + // A delete in the delta logically removes rows that are already folded + // into the cached partial result; that cannot be undone by merging, so + // fall back to a full recompute. + decision->incremental_fallback_reason = "delta contains delete predicates"; + return false; + } + // Pin the cached side for as long as the classification below reads the + // delete bitmap. The markers it looks for sit on the rowsets the cached + // snapshot was built from, and the capture above pins only the delta + // side, so nothing else keeps that evidence alive: once a compaction + // spanning both sides retires such a rowset, an overwritten row has no + // counterpart in the output and its marker is therefore not relocated + // (BaseTablet::calc_compaction_output_rowset_delete_bitmap skips a source + // row whose id does not convert), leaving the marker only on the retired + // input, which the stale sweep then hands to the unused-rowset GC. + // Classification would see no rewrite and merge the cached rows with + // their replacements: a wrong entry, stored at the current version. + // Holding the rowsets stops that, in start_delete_unused_rowset(): it + // collects a retired rowset only once nothing else references it, and + // only what it collects gets remove_rowset_delete_bitmap(), which drops + // every version of that rowset's bitmap. The other remover, the key + // ranges that the stale sweep queues onto the pre-window rowsets, is + // held off by the delta capture instead: that queue is released only + // once the swept path's own rowsets are unreferenced. + std::vector cached_side_pin; + if (merge_on_write) { + { + std::shared_lock rdlock(tablet->get_header_lock()); + auto pin_res = tablet->capture_consistent_rowsets_unlocked({0, cached_version}, + {.quiet = true}); + if (pin_res) { + cached_side_pin = std::move(pin_res.value().rowsets); + } + } + // quiet yields the prefix it walked, so cover the whole cached range + // before trusting the classification: an unpinned tail is evidence we + // cannot vouch for. + if (cached_side_pin.empty() || cached_side_pin.front()->start_version() != 0 || + cached_side_pin.back()->end_version() != cached_version) { + decision->incremental_fallback_reason = "cached versions not pinnable"; + return false; + } + // The pin is only worth anything while the classification runs, so + // let a test observe it exactly there rather than infer it. + TEST_SYNC_POINT_CALLBACK("QueryCacheRuntime::_capture_tablet_delta.cached_side_pinned", + &cached_side_pin); + } + if (merge_on_write && + _delta_rewrites_history(*tablet, *read_source, cached_version, decision->current_version)) { + // A load in the delta window marked rows of the cached snapshot as + // deleted through the delete bitmap (an upsert, a partial update or a + // delete sign hit a pre-existing key). Those rows are already folded + // into the cached partial result and cannot be subtracted, so only a + // full recompute is safe. The full run re-bases the entry at the new + // version, so an occasional backfill costs exactly one full scan and + // the next pure-append hour is incremental again. + decision->incremental_fallback_reason = "delta rewrites history rows"; + return false; + } + decision->_delta_read_sources[tablet_id] = std::move(read_source); + return true; +} + +bool QueryCacheRuntime::_delta_rewrites_history(BaseTablet& tablet, + const TabletReadSource& delta_source, + int64_t cached_version, int64_t current_version) { + RowsetIdUnorderedSet delta_rowsets; + for (const auto& split : delta_source.rs_splits) { + delta_rowsets.insert(split.rs_reader->rowset()->rowset_id()); + } + // The tablet delete bitmap is shared and it does change at versions that + // are already visible, so what keeps this scan safe is not immutability + // but the fallback it feeds. Loads add their markers before the version + // becomes visible. Compaction relocates existing markers onto its output + // rowset keeping their original version (BaseTablet::calc_compaction_ + // output_rowset_delete_bitmap); markers that matter here sit on a rowset + // at or below the cached version, and an output that swallowed such a + // rowset starts no later than it does, so the relocated marker lands + // outside the delta and is read as a rewrite: conservative, never a miss. + // (An output whose inputs all sit inside the window does belong to the + // delta, and markers on it are delta-internal, which is what we want.) The one + // rewrite that moves markers out of the window, the aggregation that + // re-stamps a stale range onto its end version (BaseTablet::agg_delete_ + // bitmap_for_stale_rowsets, run by delete_expired_stale_rowset), only + // fires once the merged versions are swept, and the capture this scan + // pairs with anchors on the window endpoints, so a window whose path is + // gone is rejected before we get here. Capture and scan are not atomic, + // so a sweep landing exactly in between (with the aggregated keys also + // already recycled) can still hide a rewrite: a known narrow race the + // fallback does not cover, shared with the baseline exact-hit path. + // Pending entries use DeleteBitmap::TEMP_VERSION_COMMON (= 0) and stay + // below the window as well. + // + // The map is ordered by (rowset, segment, version), and most entries of + // a group lie OUTSIDE the delta window (dedup history below it, pending + // entries at 0, concurrent newer loads above it), so on a long-lived + // merge-on-write tablet the map is much larger than the window. Hop from + // group to group with lower_bound/upper_bound instead of visiting every + // entry: the shared-lock hold, which excludes bitmap writers such as + // load publish, is then bounded by the in-window entries plus one + // logarithmic hop per group instead of the full map size. + // Compare versions in the uint64 key domain (both window bounds are + // non-negative), so a hypothetical out-of-int64-range stored version can + // only take the above-window hop, whose target is strictly past the + // whole group: forward progress holds for arbitrary map contents, like + // the plain walk this replaces. + const auto window_begin = static_cast(cached_version) + 1; + const auto window_end = static_cast(current_version); + const DeleteBitmap& delete_bitmap = tablet.tablet_meta()->delete_bitmap(); + std::shared_lock rdlock(delete_bitmap.lock); + const auto& bitmap_map = delete_bitmap.delete_bitmap; + auto it = bitmap_map.begin(); + while (it != bitmap_map.end()) { + const auto& [rowset_id, segment_id, version_key] = it->first; + if (version_key < window_begin) { + it = bitmap_map.lower_bound({rowset_id, segment_id, window_begin}); + continue; + } + if (version_key > window_end) { + it = bitmap_map.upper_bound( + {rowset_id, segment_id, std::numeric_limits::max()}); + continue; + } + if (!it->second.isEmpty() && !delta_rowsets.contains(rowset_id)) { + return true; + } + ++it; + } + return false; +} + } // namespace doris diff --git a/be/src/runtime/query_cache/query_cache.h b/be/src/runtime/query_cache/query_cache.h index bfa90b011e34f8..f727159eac2915 100644 --- a/be/src/runtime/query_cache/query_cache.h +++ b/be/src/runtime/query_cache/query_cache.h @@ -23,9 +23,13 @@ #include #include +#include #include +#include #include #include +#include +#include #include "common/config.h" #include "common/status.h" @@ -41,6 +45,9 @@ namespace doris { +class BaseTablet; +struct TabletReadSource; + using CacheResult = std::vector; // A handle for mid-result from query lru cache. // The handle will automatically release the cache entry when it is destroyed. @@ -73,12 +80,22 @@ class QueryCacheHandle { return *this; } + bool valid() const { return _handle != nullptr; } + std::vector* get_cache_slot_orders(); CacheResult* get_cache_result(); int64_t get_cache_version(); + // How many incremental merges have been accumulated on this entry since the + // last full recompute. See QueryCacheRuntime for the compaction policy. + int64_t get_cache_delta_count(); + + int64_t get_cache_total_bytes(); + + int64_t get_cache_total_rows(); + private: LRUCachePolicy* _cache = nullptr; Cache::Handle* _handle = nullptr; @@ -95,9 +112,28 @@ class QueryCache : public LRUCachePolicy { int64_t version; CacheResult result; std::vector slot_orders; - - CacheValue(int64_t v, CacheResult&& r, const std::vector& so) - : LRUCacheValueBase(), version(v), result(std::move(r)), slot_orders(so) {} + // Number of incremental merges accumulated on this entry since the last + // full recompute. 0 means the entry was produced by a full scan. + int64_t delta_count; + // Size of this entry, used to decide upfront whether an incremental + // merge could ever be written back under the entry_max_bytes/rows + // limits (a merged entry can only be larger than the cached one). + int64_t total_bytes; + int64_t total_rows; + + CacheValue(int64_t v, CacheResult&& r, const std::vector& so, int64_t dc = 0, + int64_t bytes = 0) + : LRUCacheValueBase(), + version(v), + result(std::move(r)), + slot_orders(so), + delta_count(dc), + total_bytes(bytes) { + total_rows = 0; + for (const auto& block : result) { + total_rows += block->rows(); + } + } }; // Create global instance of this class @@ -213,7 +249,155 @@ class QueryCache : public LRUCachePolicy { bool lookup(const CacheKey& key, int64_t version, QueryCacheHandle* handle); + // Look up the entry by key regardless of its version. The caller decides + // whether the entry is an exact hit (cached version == expected version) or + // a stale entry usable for incremental merge. Returns false if the key is + // not in the cache at all. + bool lookup_any_version(const CacheKey& key, QueryCacheHandle* handle); + void insert(const CacheKey& key, int64_t version, CacheResult& result, - const std::vector& solt_orders, int64_t cache_size); + const std::vector& solt_orders, int64_t cache_size, int64_t delta_count = 0); +}; + +// The per-fragment-instance decision of how the query cache participates in the +// execution, made exactly once (see QueryCacheRuntime) and consumed by both the +// olap scan operator and the cache source operator, so the two operators can +// never disagree (e.g. scan skips scanning because the entry looked fresh while +// cache source misses because the entry got evicted in between -- which would +// silently produce an empty result and poison the cache with it). +struct QueryCacheInstanceDecision { + enum class Mode { + // Run the full scan and (if the key is valid) write the result back. + MISS, + // The cached entry matches the current version: emit cached blocks, + // skip scanning entirely, do not write back. + HIT, + // A stale entry is reusable: scan only the delta rowsets in + // (cached_version, current_version], emit the cached blocks and the + // delta partial result side by side (the upstream merge aggregation + // combines them), then write the merged entry back. + INCREMENTAL, + }; + + ~QueryCacheInstanceDecision(); + + // Take the pre-captured delta read source of one tablet. Returns nullptr if + // absent (already taken or never captured). Only meaningful in INCREMENTAL + // mode; each tablet's read source can be consumed exactly once. + std::unique_ptr take_delta_read_source(int64_t tablet_id); + + Mode mode = Mode::MISS; + // False when build_cache_key failed (e.g. tablets in this instance carry + // different versions because FE could not align instances to partitions). + // In that case the query degrades to an uncached scan: no lookup, no write + // back, but the query itself still succeeds. + bool key_valid = false; + // False when the merged entry could never satisfy entry_max_bytes/rows + // because the reused cached entry alone already exceeds them: the query + // still scans only the delta (INCREMENTAL), but skips cloning blocks for a + // write back that would be discarded anyway. + bool write_back_feasible = true; + // Why a stale entry was not reused incrementally (empty when it was, or + // when incremental merge is not enabled for this query). For the query + // profile only. + std::string incremental_fallback_reason; + std::string cache_key; + // The version this query is reading (from the scan ranges). + int64_t current_version = 0; + // Only set in INCREMENTAL mode: the version of the reused stale entry. + int64_t cached_version = 0; + // Only set in HIT/INCREMENTAL mode: delta merges accumulated on the entry. + int64_t cached_delta_count = 0; + // Pins the cache entry in HIT/INCREMENTAL mode so it cannot be evicted (and + // its blocks cannot be freed) while this query is using it. Note the pin + // lives until the fragment is torn down; when the merged entry replaces + // this one under the same key, both stay in memory for that window and the + // LRU usage accounting only sees the new one (the mem tracker still sees + // both) -- bounded by (in-flight incremental queries) x entry size. + QueryCacheHandle handle; + +private: + friend class QueryCacheRuntime; + std::mutex _take_lock; + // INCREMENTAL mode: read sources of (cached_version, current_version] + // captured at decision time, keyed by tablet id. Captured eagerly so that a + // capture failure (e.g. the delta versions were merged away by compaction) + // downgrades the decision to MISS *before* any operator acts on it; if the + // scan discovered the failure only at prepare time, the cache source might + // already have decided to emit the stale blocks. + std::unordered_map> _delta_read_sources; }; + +// Fragment-level query cache context shared by the olap scan operator and the +// cache source operator of the same fragment. Both operators obtain the cache +// decision of their instance through get_or_make_decision(); the first caller +// makes the decision and the other one observes the same object, whatever the +// operator local-state init order is. +class QueryCacheRuntime { +public: + // `cache` is injectable for tests; production callers pass nullptr and the + // global instance is used. + explicit QueryCacheRuntime(const TQueryCacheParam& param, QueryCache* cache = nullptr) + : _param(param), _cache(cache != nullptr ? cache : QueryCache::instance()) {} + + QueryCache* cache() const { return _cache; } + + // Row-binlog scans read a different data stream and must not serve or fill + // the query cache. Called while building the operator tree (single + // threaded, before any local state init), so no locking is needed. + void disable_for_binlog_scan() { _binlog_scan = true; } + + // Idempotent: the first call for a given instance (identified by the cache + // key derived from its scan ranges) makes the decision, later calls return + // the same decision object. Never returns nullptr. + std::shared_ptr get_or_make_decision( + const std::vector& scan_ranges); + +#ifdef BE_TEST + // Tests inject a hand-crafted decision (e.g. INCREMENTAL) for an instance, + // since a real storage engine is unavailable to capture delta read sources. + void inject_decision_for_test(const std::string& cache_key, + std::shared_ptr decision) { + std::lock_guard lock(_lock); + _decisions[cache_key] = std::move(decision); + } +#endif + +private: + void _make_decision(const std::vector& scan_ranges, + QueryCacheInstanceDecision* decision); + + // Try to turn a stale entry into an INCREMENTAL decision. Returns true on + // success; on any failure the caller keeps the decision as MISS (full + // recompute), which is always safe. + bool _try_prepare_incremental(const std::vector& scan_ranges, + QueryCacheInstanceDecision* decision); + + // Validate one tablet for incremental merge and capture its delta read + // source of (cached_version, current_version]. On any failure records the + // fallback reason in the decision and returns false. + bool _capture_tablet_delta(int64_t tablet_id, int64_t cached_version, + QueryCacheInstanceDecision* decision); + + // Merge-on-write only: true if any delete-bitmap entry stamped with a + // version inside (cached_version, current_version] targets a rowset + // OUTSIDE the captured delta set, i.e. the delta window rewrote rows that + // are already folded into the cached partial result (an upsert, a partial + // update or a delete sign hit a key that predates the cached version). + // Entries targeting the delta rowsets themselves are harmless: the delta + // scan reads those rowsets with the delete bitmap applied. + static bool _delta_rewrites_history(BaseTablet& tablet, const TabletReadSource& delta_source, + int64_t cached_version, int64_t current_version); + + TQueryCacheParam _param; + QueryCache* _cache = nullptr; + bool _binlog_scan = false; + + std::mutex _lock; + std::map> _decisions; + // Shared by every instance whose cache key cannot be built (see + // get_or_make_decision): one immutable MISS decision, one log line. + std::shared_ptr _invalid_decision; +}; + } // namespace doris diff --git a/be/src/service/http/action/stream_load.cpp b/be/src/service/http/action/stream_load.cpp index 3b312a6d15367f..92cdb5844fd3a0 100644 --- a/be/src/service/http/action/stream_load.cpp +++ b/be/src/service/http/action/stream_load.cpp @@ -786,12 +786,20 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req, request.__set_group_commit_mode(ctx->group_commit_mode); } + // Keep cloud_cluster for compatibility with old FEs during rolling upgrade. New FEs use + // backend_id below to bind planning to the compute group of the receiving BE. if (!http_req->header(HTTP_COMPUTE_GROUP).empty()) { request.__set_cloud_cluster(http_req->header(HTTP_COMPUTE_GROUP)); } else if (!http_req->header(HTTP_CLOUD_CLUSTER).empty()) { request.__set_cloud_cluster(http_req->header(HTTP_CLOUD_CLUSTER)); } + if (_exec_env->cluster_info()->backend_id != 0) { + request.__set_backend_id(_exec_env->cluster_info()->backend_id); + } else { + LOG(WARNING) << "_exec_env->cluster_info not set backend_id"; + } + if (!http_req->header(HTTP_EMPTY_FIELD_AS_NULL).empty()) { if (iequal(http_req->header(HTTP_EMPTY_FIELD_AS_NULL), "true")) { request.__set_empty_field_as_null(true); diff --git a/be/src/storage/iterator/block_reader.cpp b/be/src/storage/iterator/block_reader.cpp index e96f7a0aa76800..16389c84a9ee02 100644 --- a/be/src/storage/iterator/block_reader.cpp +++ b/be/src/storage/iterator/block_reader.cpp @@ -265,6 +265,7 @@ Status BlockReader::_min_delta_next_block(Block* block, bool* eof) { RETURN_IF_ERROR(_write_binlog_op(*target_columns[target_col_idx], binlog::STREAM_CHANGE_INSERT)); } else { + // insert should use most recent value target_columns[target_col_idx]->insert_from(*_stored_data_columns[idx], group_size - 1); } @@ -277,9 +278,14 @@ Status BlockReader::_min_delta_next_block(Block* block, bool* eof) { if (idx == _binlog_op_pos) { RETURN_IF_ERROR(_write_binlog_op(*target_columns[target_col_idx], binlog::STREAM_CHANGE_DELETE)); - } else { + } else if (idx == _binlog_lsn_pos || idx == _binlog_tso_pos) { target_columns[target_col_idx]->insert_from(*_stored_data_columns[idx], group_size - 1); + } else { + // delete should use first op value + int source_idx = _resolve_source_column_index(idx, true); + target_columns[target_col_idx]->insert_from(*_stored_data_columns[source_idx], + 0); } } output_row_count++; @@ -290,9 +296,6 @@ Status BlockReader::_min_delta_next_block(Block* block, bool* eof) { if (idx == _binlog_op_pos) { RETURN_IF_ERROR(_write_binlog_op(*target_columns[target_col_idx], binlog::STREAM_CHANGE_UPDATE_BEFORE)); - } else if (idx == _binlog_lsn_pos) { - target_columns[target_col_idx]->insert_from(*_stored_data_columns[idx], - group_size - 1); } else { int source_idx = _resolve_source_column_index(idx, true); target_columns[target_col_idx]->insert_from(*_stored_data_columns[source_idx], @@ -381,7 +384,7 @@ Status BlockReader::_detail_change_next_block(Block* block, bool* eof) { output_row_count++; } else if (op == ROW_BINLOG_DELETE) { RETURN_IF_ERROR(_append_change_row(target_columns, source_block, row, - binlog::STREAM_CHANGE_DELETE, false)); + binlog::STREAM_CHANGE_DELETE, true)); output_row_count++; } diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index b34f6ce15ea26c..c13963b3bff449 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -122,7 +122,9 @@ struct TabletSize { // Storage-engine cell types, used by TabletColumn / KeyCoder and the // data_type traits chain. When adding a new value, also extend CppTypeTraits, -// FieldTypeTraits and the field_type_size() switch in storage/types.h. +// FieldTypeTraits and the field_type_size() switch in storage/types.h. Decide how it maps to +// PrimitiveType and explicitly define its behavior in primitive_type_to_storage_field_type() and +// storage_field_type_to_primitive_type(), either by providing a mapping or by throwing. enum class FieldType { OLAP_FIELD_TYPE_TINYINT = 1, // MYSQL_TYPE_TINY OLAP_FIELD_TYPE_UNSIGNED_TINYINT = 2, diff --git a/be/src/storage/segment/adaptive_block_size_predictor.cpp b/be/src/storage/segment/adaptive_block_size_predictor.cpp index d8cc700f579853..7a5ad573a2ea6c 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.cpp +++ b/be/src/storage/segment/adaptive_block_size_predictor.cpp @@ -32,11 +32,14 @@ AdaptiveBlockSizePredictor::AdaptiveBlockSizePredictor(size_t preferred_block_si _metadata_hint_bytes_per_row(metadata_hint_bytes_per_row) {} void AdaptiveBlockSizePredictor::update(const Block& block) { - size_t rows = block.rows(); + update(block.rows(), block.bytes()); +} + +void AdaptiveBlockSizePredictor::update(size_t rows, size_t bytes) { if (rows == 0) { return; } - double cur = static_cast(block.bytes()) / static_cast(rows); + double cur = static_cast(bytes) / static_cast(rows); if (!_has_history) { _bytes_per_row = cur; diff --git a/be/src/storage/segment/adaptive_block_size_predictor.h b/be/src/storage/segment/adaptive_block_size_predictor.h index e03f18c2a536d2..f327fec8517f63 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.h +++ b/be/src/storage/segment/adaptive_block_size_predictor.h @@ -60,6 +60,7 @@ class AdaptiveBlockSizePredictor { // Update EWMA estimates from a completed batch. Must be called only when block.rows() > 0 // and the batch returned Status::OK(). void update(const Block& block); + void update(size_t rows, size_t bytes); // Predict how many rows the next batch should read. // Never exceeds |block_size_rows|; never returns less than 1. diff --git a/be/src/storage/segment/column_reader.cpp b/be/src/storage/segment/column_reader.cpp index 25886d7515d2b8..db8f2d6819b04d 100644 --- a/be/src/storage/segment/column_reader.cpp +++ b/be/src/storage/segment/column_reader.cpp @@ -3037,6 +3037,12 @@ Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) { } Status DefaultValueColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, bool* has_null) { + if (!need_to_read()) { + _convert_to_place_holder_column(dst, *n); + return Status::OK(); + } + + _recovery_from_place_holder_column(dst); *has_null = _default_value_field.is_null(); _insert_many_default(dst, *n); return Status::OK(); @@ -3044,6 +3050,12 @@ Status DefaultValueColumnIterator::next_batch(size_t* n, MutableColumnPtr& dst, Status DefaultValueColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count, MutableColumnPtr& dst) { + if (!need_to_read()) { + _convert_to_place_holder_column(dst, count); + return Status::OK(); + } + + _recovery_from_place_holder_column(dst); _insert_many_default(dst, count); return Status::OK(); } diff --git a/be/src/storage/segment/column_reader.h b/be/src/storage/segment/column_reader.h index 777f8eddf0247b..327c44c4fee181 100644 --- a/be/src/storage/segment/column_reader.h +++ b/be/src/storage/segment/column_reader.h @@ -37,6 +37,7 @@ #include "common/status.h" // for Status #include "core/column/column_array.h" // ColumnArray #include "core/data_type/data_type.h" +#include "core/data_type/storage_field_type.h" #include "io/cache/cached_remote_file_reader.h" #include "io/fs/file_reader_writer_fwd.h" #include "io/io_common.h" @@ -1094,7 +1095,7 @@ class ConstantColumnReader : public ColumnReader { // when config::enable_column_type_check is on (default true), so derive the real OLAP type from // the constant value to avoid a spurious "different type between schema and column reader" error. FieldType get_meta_type() override { - return TabletColumn::get_field_type_by_type(_value.get_type()); + return primitive_type_to_storage_field_type(_value.get_type()); } Status match_condition(const AndBlockColumnPredicate* col_predicates, diff --git a/be/src/storage/segment/row_binlog_segment_writer.cpp b/be/src/storage/segment/row_binlog_segment_writer.cpp index c1102b318bb3b4..6a5737a613fad2 100644 --- a/be/src/storage/segment/row_binlog_segment_writer.cpp +++ b/be/src/storage/segment/row_binlog_segment_writer.cpp @@ -17,6 +17,9 @@ #include "storage/segment/row_binlog_segment_writer.h" +#include +#include + #include "cloud/config.h" #include "common/cast_set.h" #include "core/block/column_with_type_and_name.h" @@ -76,7 +79,7 @@ Status RowBinlogSegmentWriter::init() { _binlog_op_col_id = static_cast(op_col_id); _normal_col_start_id = tso_col_id == 0 ? BINLOG_COLNUM : 0; - uint32_t normal_col_num = cast_set(source_schema->num_visible_columns()); + uint32_t normal_col_num = cast_set(_source_data_writer->normal_column_count()); _before_col_start_id = _normal_col_start_id + normal_col_num; if (!_write_before && _tablet_schema->num_columns() > normal_col_num + BINLOG_COLNUM) { @@ -117,6 +120,7 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos, if (UNLIKELY(source_schema == nullptr)) { return Status::InternalError("binlog writer missing source_tablet_schema"); } + const size_t normal_column_count = _source_data_writer->normal_column_count(); bool is_partial_update = _binlog_opts.source.partial_update_info && _binlog_opts.source.partial_update_info->is_partial_update() && @@ -125,6 +129,7 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos, std::vector partial_cids = is_partial_update ? _binlog_opts.source.partial_update_info->update_cids : std::vector(); + std::vector row_binlog_partial_cids = partial_cids; if (is_partial_update) { if (block->columns() <= source_schema->num_key_columns() || block->columns() >= source_schema->num_columns()) { @@ -135,11 +140,12 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos, _tablet_schema->num_columns())); } - // binlog don't need invisible column - auto erase_invisible_col = std::remove_if( - partial_cids.begin(), partial_cids.end(), - [&](uint32_t cid) { return cid >= source_schema->num_visible_columns(); }); - partial_cids.erase(erase_invisible_col, partial_cids.end()); + // Partial update lists source cids. Row-binlog writes visible columns and hidden key + // columns, so hidden non-key source columns are skipped. + auto erase_non_normal_col = std::remove_if( + row_binlog_partial_cids.begin(), row_binlog_partial_cids.end(), + [&](uint32_t cid) { return !_source_data_writer->is_normal_column(cid); }); + row_binlog_partial_cids.erase(erase_non_normal_col, row_binlog_partial_cids.end()); } // get delete_sign_column from source block if has @@ -181,9 +187,9 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos, row_pos, num_rows)); } - size_t max_normal_col_id = _normal_col_start_id + source_schema->num_visible_columns(); - RETURN_IF_ERROR(_source_data_writer->fill_normal_columns(_column_writers, _normal_col_start_id, - max_normal_col_id, partial_cids)); + size_t max_normal_col_id = _normal_col_start_id + normal_column_count; + RETURN_IF_ERROR(_source_data_writer->fill_normal_columns( + _column_writers, _normal_col_start_id, max_normal_col_id, row_binlog_partial_cids)); // We read historical rows only when we really need them: // 1. partial update: build the full AFTER row. @@ -203,9 +209,14 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos, if (is_partial_update) { std::vector row_binlog_missing_column_ids; - _source_data_writer->filter_source_ids( - _binlog_opts.source.partial_update_info->missing_cids, - row_binlog_missing_column_ids); + row_binlog_missing_column_ids.reserve( + _binlog_opts.source.partial_update_info->missing_cids.size()); + // Missing cids are source cids too. Only normal columns have AFTER writers. + for (uint32_t cid : _binlog_opts.source.partial_update_info->missing_cids) { + if (_source_data_writer->is_normal_column(cid)) { + row_binlog_missing_column_ids.emplace_back(cid); + } + } // build AFTER block (fill missing columns in full_block) RETURN_IF_ERROR(_historical_data_writer->build_after_block(&full_block, row_pos, num_rows)); @@ -215,7 +226,8 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos, RETURN_IF_ERROR(after_convertor->set_source_content_with_specifid_columns( &full_block, row_pos, num_rows, row_binlog_missing_column_ids)); for (auto cid : row_binlog_missing_column_ids) { - auto converted_cid = _normal_col_start_id + cid; + auto converted_cid = + _normal_col_start_id + _source_data_writer->normal_column_ordinal(cid); auto converted_result = after_convertor->convert_column_data(cid); if (!converted_result.first.ok()) { return converted_result.first; @@ -230,9 +242,9 @@ Status RowBinlogSegmentWriter::append_block(const Block* block, size_t row_pos, DCHECK(!_tablet_schema->has_sequence_col()); // _converted_key_columns must be resized before fill binlog columns _converted_key_columns.resize(_tablet_schema->num_key_columns()); - for (size_t i = _normal_col_start_id; i < _tablet_schema->num_key_columns(); i++) { - _converted_key_columns[i] = _source_data_writer->get_converted_column( - cast_set(i - _normal_col_start_id)); + const auto& source_key_columns = _source_data_writer->source_key_columns(); + for (size_t i = 0; i < source_key_columns.size(); ++i) { + _converted_key_columns[_normal_col_start_id + i] = source_key_columns[i]; } std::vector no_operators = std::vector {}; @@ -374,7 +386,14 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) { if (UNLIKELY(source_schema == nullptr)) { return Status::InternalError("row binlog writer missing source_tablet_schema"); } - size_t value_column_num = source_schema->num_visible_value_columns(); + std::vector value_cids; + for (uint32_t cid = 0; cid < source_schema->num_columns(); ++cid) { + const auto& column = source_schema->column(cid); + if (column.visible() && !column.is_key()) { + value_cids.emplace_back(cid); + } + } + size_t value_column_num = value_cids.size(); if (value_column_num == 0) { // No BEFORE columns in row binlog schema. return Status::OK(); @@ -401,13 +420,6 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) { } else { DCHECK(_historical_data_writer != nullptr); - std::vector value_cids; - uint32_t value_start = cast_set(source_schema->num_key_columns()); - uint32_t value_end = cast_set(source_schema->num_visible_columns()); - for (uint32_t cid = value_start; cid < value_end; ++cid) { - value_cids.emplace_back(cid); - } - DCHECK_EQ(before_cids.size(), value_cids.size()); RETURN_IF_ERROR(_historical_data_writer->build_before_block(&before_block, value_cids, 0, num_rows)); @@ -429,18 +441,35 @@ Status RowBinlogSegmentWriter::_fill_before_columns(size_t num_rows) { return Status::OK(); } +bool RowBinlogSourceDataWriter::is_normal_column(uint32_t source_cid) const { + return std::find(_normal_column_ids.begin(), _normal_column_ids.end(), source_cid) != + _normal_column_ids.end(); +} + +size_t RowBinlogSourceDataWriter::normal_column_ordinal(uint32_t source_cid) const { + auto it = std::find(_normal_column_ids.begin(), _normal_column_ids.end(), source_cid); + DCHECK(it != _normal_column_ids.end()) << source_cid; + return cast_set(std::distance(_normal_column_ids.begin(), it)); +} + Status RowBinlogSourceDataWriter::init() { _olap_data_convertor = std::make_unique(); - // _normal_column_ids: the columns which we need to write into binlog from source block if (UNLIKELY(_opt.source.tablet_schema == nullptr)) { return Status::InternalError("row binlog writer missing source_tablet_schema"); } - for (uint32_t i = 0; i < _opt.source.tablet_schema->num_visible_columns(); i++) { - _normal_column_ids.emplace_back(i); + + const auto& source_schema = _opt.source.tablet_schema; + _normal_column_ids.clear(); + for (uint32_t cid = 0; cid < source_schema->num_columns(); ++cid) { + const auto& column = source_schema->column(cid); + if (!column.visible() && !column.is_key()) { + continue; + } + _normal_column_ids.emplace_back(cid); } - _olap_data_convertor->reserve(_opt.source.tablet_schema->num_columns()); - for (size_t cid = 0; cid < _opt.source.tablet_schema->num_columns(); cid++) { - _olap_data_convertor->add_column_data_convertor(_opt.source.tablet_schema->column(cid)); + _olap_data_convertor->reserve(source_schema->num_columns()); + for (size_t cid = 0; cid < source_schema->num_columns(); cid++) { + _olap_data_convertor->add_column_data_convertor(source_schema->column(cid)); } return Status::OK(); } @@ -448,31 +477,43 @@ Status RowBinlogSourceDataWriter::init() { Status RowBinlogSourceDataWriter::prepare_by_source_block( const Block* block, size_t row_pos, size_t num_rows, std::vector& partial_source_cids, Block* full_block) { - _converted_columns.resize(_normal_column_ids.size()); + TabletSchemaSPtr tablet_schema = _opt.source.tablet_schema; + // OlapBlockDataConvertor and historical lookup are indexed by source cid, so keep + // converted columns sparse over the source schema instead of row-binlog normal order. + _converted_columns.assign(tablet_schema->num_columns(), nullptr); // LOG(INFO) << block->dump_data(0, num_rows); // convert column data from engine format to storage layer format size_t col_pos_in_block = 0; - TabletSchemaSPtr tablet_schema = _opt.source.tablet_schema; - const auto& including_cids = - partial_source_cids.empty() ? _normal_column_ids : partial_source_cids; - for (auto& cid : including_cids) { - const ColumnWithTypeAndName& col = block->get_by_position(col_pos_in_block++); + const bool is_partial_update = !partial_source_cids.empty(); + const size_t column_count = + is_partial_update ? partial_source_cids.size() : _normal_column_ids.size(); + for (size_t ordinal = 0; ordinal < column_count; ++ordinal) { + const uint32_t source_cid = + is_partial_update ? partial_source_cids[ordinal] : _normal_column_ids[ordinal]; + DCHECK_LT(source_cid, tablet_schema->num_columns()); + const ColumnWithTypeAndName& col = + block->get_by_position(is_partial_update ? col_pos_in_block++ : source_cid); RETURN_IF_ERROR(_olap_data_convertor->set_source_content_with_specifid_column( - col, row_pos, num_rows, cid)); + col, row_pos, num_rows, source_cid)); // olap data convertor alway start from id = 0 - auto converted_result = _olap_data_convertor->convert_column_data(cid); + auto converted_result = _olap_data_convertor->convert_column_data(source_cid); if (!converted_result.first.ok()) { return converted_result.first; } - _converted_columns[cid] = converted_result.second; + _converted_columns[source_cid] = converted_result.second; - if (cid < tablet_schema->num_key_columns()) { - _key_columns.push_back(converted_result.second); + full_block->replace_by_position(source_cid, col.column); + } + for (uint32_t cid : _normal_column_ids) { + if (!tablet_schema->column(cid).is_key()) { + continue; } - full_block->replace_by_position(cid, col.column); + DCHECK_LT(cid, _converted_columns.size()); + DCHECK(_converted_columns[cid] != nullptr); + _key_columns.push_back(_converted_columns[cid]); } _num_rows = num_rows; @@ -497,15 +538,20 @@ Status RowBinlogSourceDataWriter::fill_normal_columns( std::vector& partial_source_cids) { DCHECK_EQ(end - start, _normal_column_ids.size()); - const auto& including_cids = - partial_source_cids.empty() ? _normal_column_ids : partial_source_cids; - for (size_t cid : including_cids) { - DCHECK(column_writers[start + cid]->get_column()->type() == + const bool is_partial_update = !partial_source_cids.empty(); + const size_t column_count = + is_partial_update ? partial_source_cids.size() : _normal_column_ids.size(); + for (size_t ordinal = 0; ordinal < column_count; ++ordinal) { + uint32_t cid = + is_partial_update ? partial_source_cids[ordinal] : _normal_column_ids[ordinal]; + size_t normal_ordinal = is_partial_update ? normal_column_ordinal(cid) : ordinal; + DCHECK(column_writers[start + normal_ordinal]->get_column()->type() == _opt.source.tablet_schema->columns()[cid]->type()) << cid; - RETURN_IF_ERROR(column_writers[start + cid]->append(_converted_columns[cid]->get_nullmap(), - _converted_columns[cid]->get_data(), - _num_rows)); + DCHECK(_converted_columns[cid] != nullptr); + RETURN_IF_ERROR(column_writers[start + normal_ordinal]->append( + _converted_columns[cid]->get_nullmap(), _converted_columns[cid]->get_data(), + _num_rows)); } return Status::OK(); diff --git a/be/src/storage/segment/row_binlog_segment_writer.h b/be/src/storage/segment/row_binlog_segment_writer.h index 3438bff6b7b0f4..69023d8a3a79c6 100644 --- a/be/src/storage/segment/row_binlog_segment_writer.h +++ b/be/src/storage/segment/row_binlog_segment_writer.h @@ -44,8 +44,6 @@ class RowBinlogSourceDataWriter { void clear(); - IOlapColumnDataAccessor* get_converted_column(uint32_t cid) { return _converted_columns[cid]; } - bool need_before() const { return _opt.write_before; } const std::vector& source_key_columns() const { return _key_columns; } @@ -53,11 +51,11 @@ class RowBinlogSourceDataWriter { std::unique_ptr& olap_data_convertor() { return _olap_data_convertor; } - void filter_source_ids(std::vector& full_cids, std::vector& res_cids) { - res_cids.reserve(full_cids.size()); - std::set_intersection(_normal_column_ids.begin(), _normal_column_ids.end(), - full_cids.begin(), full_cids.end(), std::back_inserter(res_cids)); - } + size_t normal_column_count() const { return _normal_column_ids.size(); } + + bool is_normal_column(uint32_t source_cid) const; + + size_t normal_column_ordinal(uint32_t source_cid) const; private: const SegmentWriteBinlogOptions& _opt; diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 6f4ecef1140d67..aa480f218c58a7 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -1030,18 +1030,6 @@ Status SegmentWriter::finalize(uint64_t* segment_file_size, uint64_t* index_size LOG(INFO) << "segment flush consumes a lot time_ns " << timer.elapsed_time() << ", segmemt_size " << *segment_file_size; } - // When the cache type is not ttl(expiration time == 0), the data should be split into normal cache queue - // and index cache queue - if (auto* cache_builder = _file_writer->cache_builder(); cache_builder != nullptr && - cache_builder->_expiration_time == 0 && - config::is_cloud_mode()) { - auto index_start = _index_file_cache_info.cache_start_offset(); - auto size = *index_size + *segment_file_size; - auto holder = cache_builder->allocate_cache_holder(index_start, size, _tablet->tablet_id()); - for (auto& segment : holder->file_blocks) { - static_cast(segment->change_cache_type(io::FileCacheType::INDEX)); - } - } return Status::OK(); } diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index 3158b229837890..8dec9913f8815c 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -55,134 +55,6 @@ #include "util/json/path_in_data.h" namespace doris { -FieldType TabletColumn::get_field_type_by_type(PrimitiveType primitiveType) { - switch (primitiveType) { - case PrimitiveType::INVALID_TYPE: - return FieldType::OLAP_FIELD_TYPE_UNKNOWN; - case PrimitiveType::TYPE_NULL: - return FieldType::OLAP_FIELD_TYPE_NONE; - case PrimitiveType::TYPE_BOOLEAN: - return FieldType::OLAP_FIELD_TYPE_BOOL; - case PrimitiveType::TYPE_TINYINT: - return FieldType::OLAP_FIELD_TYPE_TINYINT; - case PrimitiveType::TYPE_SMALLINT: - return FieldType::OLAP_FIELD_TYPE_SMALLINT; - case PrimitiveType::TYPE_INT: - return FieldType::OLAP_FIELD_TYPE_INT; - case PrimitiveType::TYPE_BIGINT: - return FieldType::OLAP_FIELD_TYPE_BIGINT; - case PrimitiveType::TYPE_LARGEINT: - return FieldType::OLAP_FIELD_TYPE_LARGEINT; - case PrimitiveType::TYPE_FLOAT: - return FieldType::OLAP_FIELD_TYPE_FLOAT; - case PrimitiveType::TYPE_DOUBLE: - return FieldType::OLAP_FIELD_TYPE_DOUBLE; - case PrimitiveType::TYPE_VARCHAR: - return FieldType::OLAP_FIELD_TYPE_VARCHAR; - case PrimitiveType::TYPE_DATE: - return FieldType::OLAP_FIELD_TYPE_DATE; - case PrimitiveType::TYPE_DATETIME: - return FieldType::OLAP_FIELD_TYPE_DATETIME; - case PrimitiveType::TYPE_BINARY: - return FieldType::OLAP_FIELD_TYPE_UNKNOWN; // Not implemented - case PrimitiveType::TYPE_CHAR: - return FieldType::OLAP_FIELD_TYPE_CHAR; - case PrimitiveType::TYPE_STRUCT: - return FieldType::OLAP_FIELD_TYPE_STRUCT; - case PrimitiveType::TYPE_ARRAY: - return FieldType::OLAP_FIELD_TYPE_ARRAY; - case PrimitiveType::TYPE_MAP: - return FieldType::OLAP_FIELD_TYPE_MAP; - case PrimitiveType::TYPE_HLL: - return FieldType::OLAP_FIELD_TYPE_HLL; - case PrimitiveType::TYPE_DECIMALV2: - return FieldType::OLAP_FIELD_TYPE_UNKNOWN; // Not implemented - case PrimitiveType::TYPE_BITMAP: - return FieldType::OLAP_FIELD_TYPE_BITMAP; - case PrimitiveType::TYPE_STRING: - return FieldType::OLAP_FIELD_TYPE_STRING; - case PrimitiveType::TYPE_QUANTILE_STATE: - return FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE; - case PrimitiveType::TYPE_DATEV2: - return FieldType::OLAP_FIELD_TYPE_DATEV2; - case PrimitiveType::TYPE_DATETIMEV2: - return FieldType::OLAP_FIELD_TYPE_DATETIMEV2; - case PrimitiveType::TYPE_TIMESTAMPTZ: - return FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ; - case PrimitiveType::TYPE_TIMEV2: - return FieldType::OLAP_FIELD_TYPE_TIMEV2; - case PrimitiveType::TYPE_DECIMAL32: - return FieldType::OLAP_FIELD_TYPE_DECIMAL32; - case PrimitiveType::TYPE_DECIMAL64: - return FieldType::OLAP_FIELD_TYPE_DECIMAL64; - case PrimitiveType::TYPE_DECIMAL128I: - return FieldType::OLAP_FIELD_TYPE_DECIMAL128I; - case PrimitiveType::TYPE_DECIMAL256: - return FieldType::OLAP_FIELD_TYPE_DECIMAL256; - case PrimitiveType::TYPE_JSONB: - return FieldType::OLAP_FIELD_TYPE_JSONB; - case PrimitiveType::TYPE_VARIANT: - return FieldType::OLAP_FIELD_TYPE_VARIANT; - case PrimitiveType::TYPE_IPV4: - return FieldType::OLAP_FIELD_TYPE_IPV4; - case PrimitiveType::TYPE_IPV6: - return FieldType::OLAP_FIELD_TYPE_IPV6; - case PrimitiveType::TYPE_AGG_STATE: - return FieldType::OLAP_FIELD_TYPE_AGG_STATE; - default: - return FieldType::OLAP_FIELD_TYPE_UNKNOWN; - } -} - -PrimitiveType TabletColumn::get_primitive_type_by_field_type(FieldType type) { - static const PrimitiveType mapping[] = { - /* 0 */ PrimitiveType::INVALID_TYPE, - /* 1 OLAP_FIELD_TYPE_TINYINT */ PrimitiveType::TYPE_TINYINT, - /* 2 OLAP_FIELD_TYPE_UNSIGNED_TINYINT */ PrimitiveType::INVALID_TYPE, - /* 3 OLAP_FIELD_TYPE_SMALLINT */ PrimitiveType::TYPE_SMALLINT, - /* 4 OLAP_FIELD_TYPE_UNSIGNED_SMALLINT */ PrimitiveType::INVALID_TYPE, - /* 5 OLAP_FIELD_TYPE_INT */ PrimitiveType::TYPE_INT, - /* 6 OLAP_FIELD_TYPE_UNSIGNED_INT */ PrimitiveType::INVALID_TYPE, - /* 7 OLAP_FIELD_TYPE_BIGINT */ PrimitiveType::TYPE_BIGINT, - /* 8 OLAP_FIELD_TYPE_UNSIGNED_BIGINT */ PrimitiveType::INVALID_TYPE, - /* 9 OLAP_FIELD_TYPE_LARGEINT */ PrimitiveType::TYPE_LARGEINT, - /* 10 OLAP_FIELD_TYPE_FLOAT */ PrimitiveType::TYPE_FLOAT, - /* 11 OLAP_FIELD_TYPE_DOUBLE */ PrimitiveType::TYPE_DOUBLE, - /* 12 OLAP_FIELD_TYPE_DISCRETE_DOUBLE */ PrimitiveType::INVALID_TYPE, - /* 13 OLAP_FIELD_TYPE_CHAR */ PrimitiveType::TYPE_CHAR, - /* 14 OLAP_FIELD_TYPE_DATE */ PrimitiveType::TYPE_DATE, - /* 15 OLAP_FIELD_TYPE_DATETIME */ PrimitiveType::TYPE_DATETIME, - /* 16 OLAP_FIELD_TYPE_DECIMAL */ PrimitiveType::INVALID_TYPE, - /* 17 OLAP_FIELD_TYPE_VARCHAR */ PrimitiveType::TYPE_VARCHAR, - /* 18 OLAP_FIELD_TYPE_STRUCT */ PrimitiveType::TYPE_STRUCT, - /* 19 OLAP_FIELD_TYPE_ARRAY */ PrimitiveType::TYPE_ARRAY, - /* 20 OLAP_FIELD_TYPE_MAP */ PrimitiveType::TYPE_MAP, - /* 21 OLAP_FIELD_TYPE_UNKNOWN */ PrimitiveType::INVALID_TYPE, - /* 22 OLAP_FIELD_TYPE_NONE */ PrimitiveType::TYPE_NULL, - /* 23 OLAP_FIELD_TYPE_HLL */ PrimitiveType::TYPE_HLL, - /* 24 OLAP_FIELD_TYPE_BOOL */ PrimitiveType::TYPE_BOOLEAN, - /* 25 OLAP_FIELD_TYPE_BITMAP */ PrimitiveType::TYPE_BITMAP, - /* 26 OLAP_FIELD_TYPE_STRING */ PrimitiveType::TYPE_STRING, - /* 27 OLAP_FIELD_TYPE_QUANTILE_STATE */ PrimitiveType::TYPE_QUANTILE_STATE, - /* 28 OLAP_FIELD_TYPE_DATEV2 */ PrimitiveType::TYPE_DATEV2, - /* 29 OLAP_FIELD_TYPE_DATETIMEV2 */ PrimitiveType::TYPE_DATETIMEV2, - /* 30 OLAP_FIELD_TYPE_TIMEV2 */ PrimitiveType::TYPE_TIMEV2, - /* 31 OLAP_FIELD_TYPE_DECIMAL32 */ PrimitiveType::TYPE_DECIMAL32, - /* 32 OLAP_FIELD_TYPE_DECIMAL64 */ PrimitiveType::TYPE_DECIMAL64, - /* 33 OLAP_FIELD_TYPE_DECIMAL128I */ PrimitiveType::TYPE_DECIMAL128I, - /* 34 OLAP_FIELD_TYPE_JSONB */ PrimitiveType::TYPE_JSONB, - /* 35 OLAP_FIELD_TYPE_VARIANT */ PrimitiveType::TYPE_VARIANT, - /* 36 OLAP_FIELD_TYPE_AGG_STATE */ PrimitiveType::TYPE_AGG_STATE, - /* 37 OLAP_FIELD_TYPE_DECIMAL256 */ PrimitiveType::TYPE_DECIMAL256, - /* 38 OLAP_FIELD_TYPE_IPV4 */ PrimitiveType::TYPE_IPV4, - /* 39 OLAP_FIELD_TYPE_IPV6 */ PrimitiveType::TYPE_IPV6, - /* 40 OLAP_FIELD_TYPE_TIMESTAMPTZ */ PrimitiveType::TYPE_TIMESTAMPTZ, - }; - - int idx = static_cast(type); - return mapping[idx]; -} - FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) { std::string upper_type_str = type_str; std::transform(type_str.begin(), type_str.end(), upper_type_str.begin(), @@ -238,6 +110,8 @@ FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) { } else if (0 == upper_type_str.compare("DECIMAL256")) { type = FieldType::OLAP_FIELD_TYPE_DECIMAL256; } else if (0 == upper_type_str.compare(0, 7, "DECIMAL")) { + // Keep this generic prefix match after all specific DECIMAL types; otherwise DECIMAL32, + // DECIMAL64, DECIMAL128I, and DECIMAL256 would all be classified as DECIMAL. type = FieldType::OLAP_FIELD_TYPE_DECIMAL; } else if (0 == upper_type_str.compare(0, 7, "VARCHAR")) { type = FieldType::OLAP_FIELD_TYPE_VARCHAR; diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h index bf4598e2bb94fd..caa7c9b54f6c42 100644 --- a/be/src/storage/tablet/tablet_schema.h +++ b/be/src/storage/tablet/tablet_schema.h @@ -180,8 +180,6 @@ class TabletColumn : public MetadataAdder { static std::string get_string_by_field_type(FieldType type); static std::string get_string_by_aggregation_type(FieldAggregationMethod aggregation_type); static FieldType get_field_type_by_string(const std::string& str); - static FieldType get_field_type_by_type(PrimitiveType type); - static PrimitiveType get_primitive_type_by_field_type(FieldType type); static FieldAggregationMethod get_aggregation_type_by_string(const std::string& str); static uint32_t get_field_length_by_type(TPrimitiveType::type type, uint32_t string_length); bool is_row_store_column() const; diff --git a/be/src/udf/python/python_udaf_client.cpp b/be/src/udf/python/python_udaf_client.cpp index 4308750ccec0e7..d334017415bb4a 100644 --- a/be/src/udf/python/python_udaf_client.cpp +++ b/be/src/udf/python/python_udaf_client.cpp @@ -30,6 +30,7 @@ #include "common/compiler_util.h" #include "common/status.h" #include "format/arrow/arrow_utils.h" +#include "runtime/exec_env.h" #include "udf/python/python_udf_meta.h" #include "udf/python/python_udf_runtime.h" #include "util/unaligned.h" @@ -509,7 +510,7 @@ Status PythonUDAFClient::_create_data_request_batch(const arrow::RecordBatch& in columns.push_back(input_data.column(num_input_columns - 1)); } else { // Create NULL places column for single-place mode - arrow::Int64Builder places_builder; + arrow::Int64Builder places_builder(ExecEnv::GetInstance()->arrow_memory_pool()); std::shared_ptr places_array; RETURN_DORIS_STATUS_IF_ERROR(places_builder.AppendNulls(input_data.num_rows())); RETURN_DORIS_STATUS_IF_ERROR(places_builder.Finish(&places_array)); @@ -517,7 +518,7 @@ Status PythonUDAFClient::_create_data_request_batch(const arrow::RecordBatch& in } // Add NULL binary_data column - arrow::BinaryBuilder binary_builder; + arrow::BinaryBuilder binary_builder(ExecEnv::GetInstance()->arrow_memory_pool()); std::shared_ptr binary_array; RETURN_DORIS_STATUS_IF_ERROR(binary_builder.AppendNulls(input_data.num_rows())); RETURN_DORIS_STATUS_IF_ERROR(binary_builder.Finish(&binary_array)); @@ -538,7 +539,7 @@ Status PythonUDAFClient::_create_binary_request_batch( for (int i = 0; i < num_data_columns; ++i) { std::unique_ptr builder; std::shared_ptr null_array; - RETURN_DORIS_STATUS_IF_ERROR(arrow::MakeBuilder(arrow::default_memory_pool(), + RETURN_DORIS_STATUS_IF_ERROR(arrow::MakeBuilder(ExecEnv::GetInstance()->arrow_memory_pool(), _schema->field(i)->type(), &builder)); RETURN_DORIS_STATUS_IF_ERROR(builder->AppendNull()); RETURN_DORIS_STATUS_IF_ERROR(builder->Finish(&null_array)); @@ -546,7 +547,7 @@ Status PythonUDAFClient::_create_binary_request_batch( } // Create binary_data column - arrow::BinaryBuilder binary_builder; + arrow::BinaryBuilder binary_builder(ExecEnv::GetInstance()->arrow_memory_pool()); std::shared_ptr binary_array; RETURN_DORIS_STATUS_IF_ERROR( binary_builder.Append(binary_data->data(), static_cast(binary_data->size()))); @@ -571,8 +572,8 @@ Status PythonUDAFClient::_get_empty_request_batch(std::shared_ptrfield(i); std::unique_ptr builder; std::shared_ptr null_array; - RETURN_DORIS_STATUS_IF_ERROR( - arrow::MakeBuilder(arrow::default_memory_pool(), field->type(), &builder)); + RETURN_DORIS_STATUS_IF_ERROR(arrow::MakeBuilder(ExecEnv::GetInstance()->arrow_memory_pool(), + field->type(), &builder)); RETURN_DORIS_STATUS_IF_ERROR(builder->AppendNull()); RETURN_DORIS_STATUS_IF_ERROR(builder->Finish(&null_array)); columns.push_back(null_array); diff --git a/be/src/util/bit_packing.inline.h b/be/src/util/bit_packing.inline.h index b4e0805d8aebe0..d0de57a836b4be 100644 --- a/be/src/util/bit_packing.inline.h +++ b/be/src/util/bit_packing.inline.h @@ -20,6 +20,9 @@ #include #include "util/bit_packing.h" +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +#include "util/pdep_unpack.h" +#endif namespace doris { inline int64_t BitPacking::NumValuesToUnpack(int bit_width, int64_t in_bytes, int64_t num_values) { @@ -83,8 +86,25 @@ std::pair BitPacking::UnpackValues(const uint8_t* __res const uint8_t* in_pos = in; OutType* out_pos = out; +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) + constexpr int MIN_PDEP_BATCHES = 4; + int64_t batches_read = 0; + if constexpr (PdepUnpack::should_use()) { + if (batches_to_read >= MIN_PDEP_BATCHES && PdepUnpack::is_supported()) { + for (; batches_read < batches_to_read; ++batches_read) { + PdepUnpack::unpack32(in_pos, out_pos); + in_pos += (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT; + out_pos += BATCH_SIZE; + in_bytes -= (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT; + } + } + } +#else + constexpr int64_t batches_read = 0; +#endif + // First unpack as many full batches as possible. - for (int64_t i = 0; i < batches_to_read; ++i) { + for (int64_t i = batches_read; i < batches_to_read; ++i) { in_pos = Unpack32Values(in_pos, in_bytes, out_pos); out_pos += BATCH_SIZE; in_bytes -= (BATCH_SIZE * BIT_WIDTH) / CHAR_BIT; diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp index abb64cff9e6a71..47c67a011a1701 100644 --- a/be/src/util/block_compression.cpp +++ b/be/src/util/block_compression.cpp @@ -769,11 +769,19 @@ class SnappyBlockCompression : public BlockCompressionCodec { } Status decompress(const Slice& input, Slice* output) override { + size_t uncompressed_size = 0; + if (!snappy::GetUncompressedLength(input.data, input.size, &uncompressed_size)) { + return Status::InvalidArgument("Fail to get Snappy uncompressed length"); + } + // RawUncompress has no capacity argument, so reject an undersized destination first. + if (uncompressed_size > output->size) { + return Status::InvalidArgument("Snappy output size {} exceeds buffer capacity {}", + uncompressed_size, output->size); + } if (!snappy::RawUncompress(input.data, input.size, output->data)) { return Status::InvalidArgument("Fail to do Snappy decompress"); } - // NOTE: GetUncompressedLength only takes O(1) time - snappy::GetUncompressedLength(input.data, input.size, &output->size); + output->size = uncompressed_size; return Status::OK(); } diff --git a/be/src/util/blocking_queue.hpp b/be/src/util/blocking_queue.hpp index dbf59c341b1cd6..0052f8dae24967 100644 --- a/be/src/util/blocking_queue.hpp +++ b/be/src/util/blocking_queue.hpp @@ -30,6 +30,10 @@ #include "common/logging.h" #include "util/stopwatch.hpp" +#ifdef BE_TEST +#include "cpp/sync_point.h" +#endif + namespace doris { // Fixed capacity FIFO queue, where both BlockingGet and BlockingPut operations block // if the queue is empty or full, respectively. @@ -56,21 +60,26 @@ class BlockingQueue { MonotonicStopWatch timer; timer.start(); std::unique_lock unique_lock(_lock); +#ifdef BE_TEST + TEST_SYNC_POINT("BlockingQueue::controlled_blocking_get::after_lock"); +#endif while (!(_shutdown || !_list.empty())) { ++_get_waiting; - if (_get_cv.wait_for(unique_lock, std::chrono::milliseconds(cv_wait_timeout_ms)) == - std::cv_status::timeout) { - _get_waiting--; - } +#ifdef BE_TEST + TEST_SYNC_POINT("BlockingQueue::controlled_blocking_get::before_wait"); +#endif + _get_cv.wait_for(unique_lock, std::chrono::milliseconds(cv_wait_timeout_ms)); + DCHECK_GT(_get_waiting, 0); + --_get_waiting; } _total_get_wait_time += timer.elapsed_time(); if (!_list.empty()) { *out = _list.front(); _list.pop_front(); - if (_put_waiting > 0) { - --_put_waiting; - unique_lock.unlock(); + const bool has_put_waiter = _put_waiting > 0; + unique_lock.unlock(); + if (has_put_waiter) { _put_cv.notify_one(); } return true; @@ -93,10 +102,12 @@ class BlockingQueue { std::unique_lock unique_lock(_lock); while (!(_shutdown || _list.size() < _max_elements)) { ++_put_waiting; - if (_put_cv.wait_for(unique_lock, std::chrono::milliseconds(cv_wait_timeout_ms)) == - std::cv_status::timeout) { - _put_waiting--; - } +#ifdef BE_TEST + TEST_SYNC_POINT("BlockingQueue::controlled_blocking_put::before_wait"); +#endif + _put_cv.wait_for(unique_lock, std::chrono::milliseconds(cv_wait_timeout_ms)); + DCHECK_GT(_put_waiting, 0); + --_put_waiting; } _total_put_wait_time += timer.elapsed_time(); @@ -105,9 +116,9 @@ class BlockingQueue { } _list.push_back(val); - if (_get_waiting > 0) { - --_get_waiting; - unique_lock.unlock(); + const bool has_get_waiter = _get_waiting > 0; + unique_lock.unlock(); + if (has_get_waiter) { _get_cv.notify_one(); } return true; @@ -115,13 +126,12 @@ class BlockingQueue { // Return false if queue full or has been shutdown. bool try_put(const T& val) { - if (_shutdown || _list.size() >= _max_elements) { - return false; - } - MonotonicStopWatch timer; timer.start(); std::unique_lock unique_lock(_lock); +#ifdef BE_TEST + TEST_SYNC_POINT("BlockingQueue::try_put::after_lock"); +#endif _total_put_wait_time += timer.elapsed_time(); if (_shutdown || _list.size() >= _max_elements) { @@ -129,9 +139,9 @@ class BlockingQueue { } _list.push_back(val); - if (_get_waiting > 0) { - --_get_waiting; - unique_lock.unlock(); + const bool has_get_waiter = _get_waiting > 0; + unique_lock.unlock(); + if (has_get_waiter) { _get_cv.notify_one(); } return true; @@ -155,6 +165,18 @@ class BlockingQueue { uint32_t get_capacity() const { return _max_elements; } +#ifdef BE_TEST + size_t get_waiting_count_for_test() const { + std::lock_guard guard(_lock); + return _get_waiting; + } + + size_t put_waiting_count_for_test() const { + std::lock_guard guard(_lock); + return _put_waiting; + } +#endif + // Returns the total amount of time threads have blocked in BlockingGet. uint64_t total_get_wait_time() const { return _total_get_wait_time; } @@ -167,11 +189,14 @@ class BlockingQueue { const int _max_elements; std::condition_variable _get_cv; // 'get' callers wait on this std::condition_variable _put_cv; // 'put' callers wait on this - // _lock guards access to _list, total_get_wait_time, and total_put_wait_time + // _lock guards access to _shutdown, _get_waiting, _put_waiting, _list, total_get_wait_time, and total_put_wait_time mutable std::mutex _lock; std::list _list; std::atomic _total_get_wait_time; std::atomic _total_put_wait_time; + // Number of threads currently inside the corresponding wait_for() call. The waiter that + // increments a counter is also responsible for decrementing it after every wakeup reason. + // Notifiers only inspect these counters and never consume a waiter's registration. size_t _get_waiting; size_t _put_waiting; }; diff --git a/be/src/util/coding.h b/be/src/util/coding.h index 48a2a8532b0610..26d6a2f827c95f 100644 --- a/be/src/util/coding.h +++ b/be/src/util/coding.h @@ -71,6 +71,21 @@ inline uint64_t decode_fixed64_le(const uint8_t* buf) { return to_endian(res); } +inline void decode_fixed64_le_array(uint64_t* dst, const void* src, size_t n) { + if (n == 0) { + return; + } + if constexpr (std::endian::native == std::endian::little) { + memcpy(dst, src, sizeof(uint64_t) * n); + } else { + const auto* ptr = reinterpret_cast(src); + for (size_t i = 0; i < n; ++i) { + dst[i] = decode_fixed64_le(ptr); + ptr += sizeof(uint64_t); + } + } +} + inline uint128_t decode_fixed128_le(const uint8_t* buf) { uint128_t res; memcpy(&res, buf, sizeof(res)); diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp index 400c8f9381b77c..8c24a886acb2b4 100644 --- a/be/src/util/cpu_info.cpp +++ b/be/src/util/cpu_info.cpp @@ -377,6 +377,17 @@ void CpuInfo::_get_cache_info(long cache_sizes[NUM_CACHE_LEVELS], #endif } +long CpuInfo::get_l2_cache_size() { + static const long l2_cache_size = [] { + long cache_sizes[NUM_CACHE_LEVELS] = {}; + long cache_line_sizes[NUM_CACHE_LEVELS] = {}; + _get_cache_info(cache_sizes, cache_line_sizes); + constexpr long DEFAULT_L2_CACHE_SIZE = 256 * 1024; + return cache_sizes[L2_CACHE] > 0 ? cache_sizes[L2_CACHE] : DEFAULT_L2_CACHE_SIZE; + }(); + return l2_cache_size; +} + std::string CpuInfo::debug_string() { DCHECK(initialized_); std::stringstream stream; diff --git a/be/src/util/cpu_info.h b/be/src/util/cpu_info.h index 19548ecc964659..c75b814721cc57 100644 --- a/be/src/util/cpu_info.h +++ b/be/src/util/cpu_info.h @@ -146,6 +146,10 @@ class CpuInfo { return model_name_; } + // Some virtualized hosts do not expose cache topology. Always return a positive value so hot + // decode paths can use this threshold without repeating platform-specific guards. + static long get_l2_cache_size(); + static std::string debug_string(); /// A utility class for temporarily disabling CPU features. Usage: diff --git a/be/src/util/pdep_unpack.h b/be/src/util/pdep_unpack.h new file mode 100644 index 00000000000000..10403d94e4a158 --- /dev/null +++ b/be/src/util/pdep_unpack.h @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/config.h" + +namespace doris { + +class PdepUnpack { +public: + static bool is_supported() { + return config::enable_bmi2_optimizations && __builtin_cpu_supports("bmi2") && + __builtin_cpu_supports("avx2"); + } + + template + static constexpr bool is_supported_type() { + return BIT_WIDTH > 0 && BIT_WIDTH <= std::numeric_limits::digits && + (std::is_same_v || std::is_same_v || + std::is_same_v); + } + + template + static constexpr bool should_use() { + // Keep the generic implementation available for benchmarking all supported widths, but + // only select PDEP in the production path for widths below 16. These widths can use the + // byte/word deposit layouts and the AVX2 widening specializations below. At 16 bits and + // above, unpack32() falls back to multiple generic 64-bit PDEP groups per batch and + // competes with efficient scalar specializations, including copy-like 16- and 32-bit + // cases. Benchmarks with L1-, L2-, and larger working sets show non-monotonic results and + // repeatable regressions for multiple high widths. Because the profitable high widths are + // CPU- and working-set-dependent, an irregular per-width allowlist would not be portable; + // use the scalar implementation conservatively instead. + return is_supported_type() && BIT_WIDTH < 16; + } + + template + __attribute__((target("bmi2,avx2"))) static void unpack32(const uint8_t* input, T* output) { + static_assert(is_supported_type()); + if constexpr (std::is_same_v) { + unpack32_with_pdep(input, output); + } else if constexpr (std::is_same_v && BIT_WIDTH <= 8) { + for (int group = 0; group < 2; ++group) { + const int first_value = group * 16; + const uint8_t* group_input = input + group * 2 * BIT_WIDTH; + uint64_t expanded0 = _pdep_u64(load_packed_group<0, 8 * BIT_WIDTH>(group_input), + pdep_mask()); + uint64_t expanded1 = + _pdep_u64(load_packed_group<8 * BIT_WIDTH, 8 * BIT_WIDTH>(group_input), + pdep_mask()); + __m128i packed8 = _mm_set_epi64x(static_cast(expanded1), + static_cast(expanded0)); + __m256i unpacked16 = _mm256_cvtepu8_epi16(packed8); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(output + first_value), unpacked16); + } + } else if constexpr (std::is_same_v && BIT_WIDTH <= 8) { + for (int group = 0; group < 4; ++group) { + uint64_t expanded = + _pdep_u64(load_packed_group<0, 8 * BIT_WIDTH>(input + group * BIT_WIDTH), + pdep_mask()); + __m256i unpacked32 = _mm256_cvtepu8_epi32(_mm_cvtsi64_si128(expanded)); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(output + group * 8), unpacked32); + } + } else if constexpr (std::is_same_v && BIT_WIDTH <= 15) { + for (int group = 0; group < 4; ++group) { + const int first_value = group * 8; + const uint8_t* group_input = input + group * BIT_WIDTH; + uint64_t expanded0 = _pdep_u64(load_packed_group<0, 4 * BIT_WIDTH>(group_input), + pdep_mask()); + uint64_t expanded1 = + _pdep_u64(load_packed_group<4 * BIT_WIDTH, 4 * BIT_WIDTH>(group_input), + pdep_mask()); + __m128i packed16 = _mm_set_epi64x(static_cast(expanded1), + static_cast(expanded0)); + __m256i unpacked32 = _mm256_cvtepu16_epi32(packed16); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(output + first_value), unpacked32); + } + } else { + unpack32_with_pdep(input, output); + } + } + +private: + template + static constexpr uint64_t pdep_mask() { + constexpr int lane_bits = sizeof(T) * 8; + constexpr int lanes = 64 / lane_bits; + constexpr uint64_t value_mask = (1ULL << BIT_WIDTH) - 1; + uint64_t mask = 0; + for (int lane = 0; lane < lanes; ++lane) { + mask |= value_mask << (lane * lane_bits); + } + return mask; + } + + template + static uint64_t load_packed_group(const uint8_t* input) { + constexpr int byte_offset = BIT_OFFSET / 8; + constexpr int shift = BIT_OFFSET % 8; + constexpr int bytes_needed = (shift + PACKED_BITS + 7) / 8; + static_assert(PACKED_BITS <= 64); + static_assert(bytes_needed <= 9); + + uint64_t low = 0; + std::memcpy(&low, input + byte_offset, bytes_needed < 8 ? bytes_needed : 8); + if constexpr (bytes_needed <= 8) { + return low >> shift; + } else { + uint8_t high = input[byte_offset + 8]; + return static_cast((static_cast(high) << 64 | low) >> + shift); + } + } + + template + __attribute__((target("bmi2"))) static void unpack_group(const uint8_t* input, T* output) { + constexpr int lanes = 64 / (sizeof(T) * 8); + constexpr int first_value = GROUP * lanes; + constexpr int bit_offset = first_value * BIT_WIDTH; + constexpr int packed_bits = lanes * BIT_WIDTH; + uint64_t packed = load_packed_group(input); + uint64_t expanded = _pdep_u64(packed, pdep_mask()); + std::memcpy(output + first_value, &expanded, sizeof(expanded)); + } + + template + __attribute__((target("bmi2"))) static void unpack32_with_pdep_impl( + const uint8_t* input, T* output, std::index_sequence) { + (unpack_group(input, output), ...); + } + + template + __attribute__((target("bmi2"))) static void unpack32_with_pdep(const uint8_t* input, + T* output) { + constexpr int lanes = 64 / (sizeof(T) * 8); + unpack32_with_pdep_impl(input, output, + std::make_index_sequence<32 / lanes> {}); + } +}; + +} // namespace doris + +#endif diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index fc4c81d4bd96f2..c6e86a7f4b290f 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -158,11 +158,6 @@ constexpr char S3_EXTERNAL_ID[] = "AWS_EXTERNAL_ID"; constexpr char S3_CREDENTIALS_PROVIDER_TYPE[] = "AWS_CREDENTIALS_PROVIDER_TYPE"; } // namespace -bvar::Adder get_rate_limit_ns("get_rate_limit_ns"); -bvar::Adder get_rate_limit_exceed_req_num("get_rate_limit_exceed_req_num"); -bvar::Adder put_rate_limit_ns("put_rate_limit_ns"); -bvar::Adder put_rate_limit_exceed_req_num("put_rate_limit_exceed_req_num"); - static std::atomic last_s3_get_token_bucket_tokens {0}; static std::atomic last_s3_get_token_limit {0}; static std::atomic last_s3_get_token_per_second {0}; @@ -230,6 +225,11 @@ int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_bur return S3ClientFactory::instance().rate_limiter(type)->reset(max_speed, max_burst, limit); } +int64_t apply_s3_rate_limit(S3RateLimitType type) { + return doris::apply_s3_rate_limit(type, S3ClientFactory::instance().rate_limiter(type), + config::s3_rate_limiter_log_interval); +} + S3ClientFactory::S3ClientFactory() { _aws_options = Aws::SDKOptions {}; auto logLevel = static_cast(config::aws_log_level); @@ -242,12 +242,10 @@ S3ClientFactory::S3ClientFactory() { _rate_limiters = { std::make_unique( config::s3_get_token_per_second, config::s3_get_bucket_tokens, - config::s3_get_token_limit, - metric_func_factory(get_rate_limit_ns, get_rate_limit_exceed_req_num)), + config::s3_get_token_limit, s3_rate_limiter_metric_func(S3RateLimitType::GET)), std::make_unique( config::s3_put_token_per_second, config::s3_put_bucket_tokens, - config::s3_put_token_limit, - metric_func_factory(put_rate_limit_ns, put_rate_limit_exceed_req_num))}; + config::s3_put_token_limit, s3_rate_limiter_metric_func(S3RateLimitType::PUT))}; #ifdef USE_AZURE auto azureLogLevel = @@ -349,6 +347,7 @@ std::shared_ptr S3ClientFactory::_create_azure_client( } Azure::Storage::Blobs::BlobClientOptions options; + options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests); options.Retry.MaxRetries = config::max_s3_client_retry; options.PerRetryPolicies.emplace_back(std::make_unique()); if (_ca_cert_file_path.empty()) { @@ -525,7 +524,7 @@ std::shared_ptr S3ClientFactory::_create_s3_client( } aws_config.retryStrategy = std::make_shared( - config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/false); + config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/true); std::shared_ptr new_client = std::make_shared( get_aws_credentials_provider(s3_conf), std::move(aws_config), diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 5546db857c91b1..064f88accd8539 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -64,6 +64,7 @@ extern bvar::LatencyRecorder s3_copy_object_latency; std::string hide_access_key(const std::string& ak); int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_burst, size_t limit); +int64_t apply_s3_rate_limit(S3RateLimitType type); // Rebuild the S3 GET/PUT rate limiters if the related configs have changed. // Safe to call periodically; it is a no-op when nothing changed. void check_s3_rate_limiter_config_changed(); diff --git a/be/test/cloud/cloud_tablet_query_prefer_cache_test.cpp b/be/test/cloud/cloud_tablet_query_prefer_cache_test.cpp index a0d918753756ad..39e3ac1d032d8b 100644 --- a/be/test/cloud/cloud_tablet_query_prefer_cache_test.cpp +++ b/be/test/cloud/cloud_tablet_query_prefer_cache_test.cpp @@ -22,7 +22,6 @@ #include #include -#include #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" @@ -121,11 +120,13 @@ class TestQueryPreferCache : public testing::Test { } else { tablet->add_not_warmed_up_rowset(output_rowset->rowset_id()); } - std::ranges::copy_if(std::views::values(tablet->rowset_map()), - std::back_inserter(input_rowsets), [=](const RowsetSharedPtr& rowset) { - return rowset->version().first >= start_version && - rowset->version().first <= end_version; - }); + for (const auto& kv : tablet->rowset_map()) { + const auto& rowset = kv.second; + if (rowset->version().first >= start_version && + rowset->version().first <= end_version) { + input_rowsets.push_back(rowset); + } + } if (input_rowsets.size() == 1) { tablet->add_rowsets({output_rowset}, true, wrlock); } else { diff --git a/be/test/cloud/cloud_tablet_query_with_tolerance_test.cpp b/be/test/cloud/cloud_tablet_query_with_tolerance_test.cpp index 151ec37c4ee448..1ceddf3ce4b95b 100644 --- a/be/test/cloud/cloud_tablet_query_with_tolerance_test.cpp +++ b/be/test/cloud/cloud_tablet_query_with_tolerance_test.cpp @@ -22,7 +22,6 @@ #include #include -#include #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" @@ -116,11 +115,13 @@ class TestFreshnessTolerance : public testing::Test { } else { tablet->add_not_warmed_up_rowset(output_rowset->rowset_id()); } - std::ranges::copy_if(std::views::values(tablet->rowset_map()), - std::back_inserter(input_rowsets), [=](const RowsetSharedPtr& rowset) { - return rowset->version().first >= start_version && - rowset->version().first <= end_version; - }); + for (const auto& kv : tablet->rowset_map()) { + const auto& rowset = kv.second; + if (rowset->version().first >= start_version && + rowset->version().first <= end_version) { + input_rowsets.push_back(rowset); + } + } if (input_rowsets.size() == 1) { tablet->add_rowsets({output_rowset}, true, wrlock); } else { @@ -151,11 +152,13 @@ class TestFreshnessTolerance : public testing::Test { std::vector input_rowsets; auto output_rowset = create_rowset(Version {start_version, end_version}, visible_timestamp); // Intentionally do NOT add any warmup state entry for this rowset - std::ranges::copy_if(std::views::values(tablet->rowset_map()), - std::back_inserter(input_rowsets), [=](const RowsetSharedPtr& rowset) { - return rowset->version().first >= start_version && - rowset->version().first <= end_version; - }); + for (const auto& kv : tablet->rowset_map()) { + const auto& rowset = kv.second; + if (rowset->version().first >= start_version && + rowset->version().first <= end_version) { + input_rowsets.push_back(rowset); + } + } if (input_rowsets.size() == 1) { tablet->add_rowsets({output_rowset}, true, wrlock); } else { diff --git a/be/test/cloud/cloud_tablet_test.cpp b/be/test/cloud/cloud_tablet_test.cpp index 228e7421ee370a..4c1bfbf79a3252 100644 --- a/be/test/cloud/cloud_tablet_test.cpp +++ b/be/test/cloud/cloud_tablet_test.cpp @@ -24,7 +24,6 @@ #include #include -#include #include "cloud/cloud_meta_mgr.h" #include "cloud/cloud_storage_engine.h" diff --git a/be/test/common/status_test.cpp b/be/test/common/status_test.cpp index f607e56f701db2..be555a0912c964 100644 --- a/be/test/common/status_test.cpp +++ b/be/test/common/status_test.cpp @@ -21,11 +21,33 @@ #include #include "gtest/gtest_pred_impl.h" +#include "util/debug_points.h" namespace doris { class StatusTest : public testing::Test {}; +class ScopedEmptyStackTraceFault { +public: + ScopedEmptyStackTraceFault() + : _enable_debug_points(config::enable_debug_points), + _enable_stacktrace(config::enable_stacktrace) { + config::enable_debug_points = true; + config::enable_stacktrace = true; + DebugPoints::instance()->add("StackTrace::tryCapture.empty_trace"); + } + + ~ScopedEmptyStackTraceFault() { + DebugPoints::instance()->remove("StackTrace::tryCapture.empty_trace"); + config::enable_stacktrace = _enable_stacktrace; + config::enable_debug_points = _enable_debug_points; + } + +private: + const bool _enable_debug_points; + const bool _enable_stacktrace; +}; + TEST_F(StatusTest, OK) { // default Status st; @@ -117,6 +139,13 @@ TEST_F(StatusTest, Error) { } } +TEST_F(StatusTest, EmptyCapturedStackTrace) { + ScopedEmptyStackTraceFault fault; + + Status status = Status::NotFound("fault-injected empty stack trace"); + EXPECT_NE(status.to_string().find(""), std::string::npos); +} + TEST_F(StatusTest /*unused*/, Format /*unused*/) { // status == ok Status st_ok = Status::OK(); diff --git a/be/test/core/block/block_test.cpp b/be/test/core/block/block_test.cpp index 3b29f819e093d4..2a3f07e01c5582 100644 --- a/be/test/core/block/block_test.cpp +++ b/be/test/core/block/block_test.cpp @@ -1154,6 +1154,37 @@ TEST(BlockTest, merge_impl) { EXPECT_ANY_THROW(st = mutable_block.merge_impl(std::move(block2))); } +TEST(BlockTest, MergeMaterializesConstNullableDestination) { + for (bool ignore_overflow : {false, true}) { + auto nullable_type = make_nullable(std::make_shared()); + + auto const_data = nullable_type->create_column(); + const_data->insert_default(); + Block destination; + destination.insert( + {ColumnConst::create(std::move(const_data), 2), nullable_type, "lineage"}); + + auto source_column = nullable_type->create_column(); + source_column->insert_default(); + Block source; + source.insert({std::move(source_column), nullable_type, "lineage"}); + + { + ScopedMutableBlock scoped_destination(&destination); + auto status = ignore_overflow + ? scoped_destination.mutable_block().merge_ignore_overflow(source) + : scoped_destination.mutable_block().merge(source); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + ASSERT_EQ(destination.rows(), 3); + EXPECT_FALSE(is_column_const(*destination.get_by_position(0).column)); + for (size_t i = 0; i < destination.rows(); ++i) { + EXPECT_TRUE(destination.get_by_position(0).column->is_null_at(i)); + } + } +} + TEST(BlockTest, ctor) { TDescriptorTableBuilder builder; TTupleDescriptorBuilder tuple_builder; diff --git a/be/test/core/block/column_complex_test.cpp b/be/test/core/block/column_complex_test.cpp index 3a2629474f98b0..4ced4c4f8e7105 100644 --- a/be/test/core/block/column_complex_test.cpp +++ b/be/test/core/block/column_complex_test.cpp @@ -60,10 +60,10 @@ TEST(ColumnComplexTest, GetDataAtTest) { column_bitmap_verify->insert_default(); column_hll_verify->insert_default(); column_quantile_state_verify->insert_many_defaults(1); - ASSERT_EQ(column_bitmap_verify->byte_size(), 80); + ASSERT_EQ(column_bitmap_verify->byte_size(), sizeof(BitmapValue)); ASSERT_EQ(column_hll_verify->byte_size(), 64); ASSERT_EQ(column_quantile_state_verify->byte_size(), 64); - ASSERT_EQ(column_bitmap_verify->allocated_bytes(), 80); + ASSERT_EQ(column_bitmap_verify->allocated_bytes(), sizeof(BitmapValue)); ASSERT_EQ(column_hll_verify->allocated_bytes(), 64); ASSERT_EQ(column_quantile_state_verify->allocated_bytes(), 64); std::cout << "1. test byte_size/allocated_bytes empty success" << std::endl; diff --git a/be/test/core/column/column_variant_test.cpp b/be/test/core/column/column_variant_test.cpp index 4db42b8a3efce1..24798accf9c3cd 100644 --- a/be/test/core/column/column_variant_test.cpp +++ b/be/test/core/column/column_variant_test.cpp @@ -1066,6 +1066,33 @@ TEST_F(ColumnVariantTest, test_insert_indices_from) { } } +TEST_F(ColumnVariantTest, visible_root_does_not_hide_sparse_fields) { + auto source = ColumnVariant::create(0, false); + VariantMap mixed; + mixed.try_emplace(PathInData(), FieldWithDataType {.field = get_jsonb_field("array_int")}); + mixed.try_emplace(PathInData("n"), FieldWithDataType {.field = VariantUtil::get_field("int")}); + mixed.try_emplace(PathInData("word"), + FieldWithDataType {.field = VariantUtil::get_field("string")}); + source->try_insert(Field::create_field(std::move(mixed))); + source->finalize(); + + auto destination = ColumnVariant::create(1, false); + destination->try_insert( + VariantUtil::construct_variant_map({{"k", VariantUtil::get_field("int")}})); + destination->insert_range_from(*source, 0, 1); + destination->finalize(); + + EXPECT_EQ(destination->get_subcolumns().get_root()->data.get_least_common_base_type_id(), + PrimitiveType::TYPE_JSONB); + const auto& sparse_offsets = destination->serialized_sparse_column_offsets(); + EXPECT_LT(sparse_offsets[0], sparse_offsets[1]); + + DataTypeSerDe::FormatOptions options; + std::string json; + destination->serialize_one_row_to_string(1, &json, options); + EXPECT_EQ(json, R"({"n":20,"word":"str"})"); +} + TEST_F(ColumnVariantTest, is_variable_length) { EXPECT_TRUE(column_variant->is_variable_length()); } diff --git a/be/test/core/data_type/storage_field_type_test.cpp b/be/test/core/data_type/storage_field_type_test.cpp new file mode 100644 index 00000000000000..af621ceb9c6abe --- /dev/null +++ b/be/test/core/data_type/storage_field_type_test.cpp @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/storage_field_type.h" + +#include + +#include +#include + +#include "common/exception.h" +#include "storage/olap_common.h" + +namespace doris { + +namespace { + +struct TypePair { + PrimitiveType primitive_type; + FieldType field_type; +}; + +constexpr std::array supported_mappings { + TypePair {PrimitiveType::INVALID_TYPE, FieldType::OLAP_FIELD_TYPE_UNKNOWN}, + TypePair {PrimitiveType::TYPE_NULL, FieldType::OLAP_FIELD_TYPE_NONE}, + TypePair {PrimitiveType::TYPE_BOOLEAN, FieldType::OLAP_FIELD_TYPE_BOOL}, + TypePair {PrimitiveType::TYPE_TINYINT, FieldType::OLAP_FIELD_TYPE_TINYINT}, + TypePair {PrimitiveType::TYPE_SMALLINT, FieldType::OLAP_FIELD_TYPE_SMALLINT}, + TypePair {PrimitiveType::TYPE_INT, FieldType::OLAP_FIELD_TYPE_INT}, + TypePair {PrimitiveType::TYPE_BIGINT, FieldType::OLAP_FIELD_TYPE_BIGINT}, + TypePair {PrimitiveType::TYPE_LARGEINT, FieldType::OLAP_FIELD_TYPE_LARGEINT}, + TypePair {PrimitiveType::TYPE_FLOAT, FieldType::OLAP_FIELD_TYPE_FLOAT}, + TypePair {PrimitiveType::TYPE_DOUBLE, FieldType::OLAP_FIELD_TYPE_DOUBLE}, + TypePair {PrimitiveType::TYPE_VARCHAR, FieldType::OLAP_FIELD_TYPE_VARCHAR}, + TypePair {PrimitiveType::TYPE_DATE, FieldType::OLAP_FIELD_TYPE_DATE}, + TypePair {PrimitiveType::TYPE_DATETIME, FieldType::OLAP_FIELD_TYPE_DATETIME}, + TypePair {PrimitiveType::TYPE_CHAR, FieldType::OLAP_FIELD_TYPE_CHAR}, + TypePair {PrimitiveType::TYPE_STRUCT, FieldType::OLAP_FIELD_TYPE_STRUCT}, + TypePair {PrimitiveType::TYPE_ARRAY, FieldType::OLAP_FIELD_TYPE_ARRAY}, + TypePair {PrimitiveType::TYPE_MAP, FieldType::OLAP_FIELD_TYPE_MAP}, + TypePair {PrimitiveType::TYPE_HLL, FieldType::OLAP_FIELD_TYPE_HLL}, + TypePair {PrimitiveType::TYPE_DECIMALV2, FieldType::OLAP_FIELD_TYPE_DECIMAL}, + TypePair {PrimitiveType::TYPE_BITMAP, FieldType::OLAP_FIELD_TYPE_BITMAP}, + TypePair {PrimitiveType::TYPE_STRING, FieldType::OLAP_FIELD_TYPE_STRING}, + TypePair {PrimitiveType::TYPE_QUANTILE_STATE, FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE}, + TypePair {PrimitiveType::TYPE_DATEV2, FieldType::OLAP_FIELD_TYPE_DATEV2}, + TypePair {PrimitiveType::TYPE_DATETIMEV2, FieldType::OLAP_FIELD_TYPE_DATETIMEV2}, + TypePair {PrimitiveType::TYPE_TIMEV2, FieldType::OLAP_FIELD_TYPE_TIMEV2}, + TypePair {PrimitiveType::TYPE_DECIMAL32, FieldType::OLAP_FIELD_TYPE_DECIMAL32}, + TypePair {PrimitiveType::TYPE_DECIMAL64, FieldType::OLAP_FIELD_TYPE_DECIMAL64}, + TypePair {PrimitiveType::TYPE_DECIMAL128I, FieldType::OLAP_FIELD_TYPE_DECIMAL128I}, + TypePair {PrimitiveType::TYPE_JSONB, FieldType::OLAP_FIELD_TYPE_JSONB}, + TypePair {PrimitiveType::TYPE_VARIANT, FieldType::OLAP_FIELD_TYPE_VARIANT}, + TypePair {PrimitiveType::TYPE_AGG_STATE, FieldType::OLAP_FIELD_TYPE_AGG_STATE}, + TypePair {PrimitiveType::TYPE_DECIMAL256, FieldType::OLAP_FIELD_TYPE_DECIMAL256}, + TypePair {PrimitiveType::TYPE_IPV4, FieldType::OLAP_FIELD_TYPE_IPV4}, + TypePair {PrimitiveType::TYPE_IPV6, FieldType::OLAP_FIELD_TYPE_IPV6}, + TypePair {PrimitiveType::TYPE_UINT32, FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT}, + TypePair {PrimitiveType::TYPE_UINT64, FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT}, + TypePair {PrimitiveType::TYPE_TIMESTAMPTZ, FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ}, +}; + +TEST(StorageFieldTypeTest, SupportedMappingsRoundTrip) { + for (const auto& [primitive_type, field_type] : supported_mappings) { + SCOPED_TRACE(static_cast(primitive_type)); + EXPECT_EQ(primitive_type_to_storage_field_type(primitive_type), field_type); + EXPECT_EQ(storage_field_type_to_primitive_type(field_type), primitive_type); + } +} + +TEST(StorageFieldTypeTest, UnsupportedPrimitiveTypesThrow) { + constexpr std::array unsupported_types { + PrimitiveType::TYPE_BINARY, + static_cast(14), // TYPE_DECIMAL (deprecated) + static_cast(21), // TYPE_TIME (deprecated) + static_cast(33), // TYPE_LAMBDA_FUNCTION (deprecated) + PrimitiveType::TYPE_FIXED_LENGTH_OBJECT, + PrimitiveType::TYPE_VARBINARY, + static_cast(43), + static_cast(255), + }; + + for (const auto type : unsupported_types) { + SCOPED_TRACE(static_cast(type)); + EXPECT_THROW((void)primitive_type_to_storage_field_type(type), Exception); + } +} + +TEST(StorageFieldTypeTest, UnsupportedOrInvalidFieldTypesThrow) { + constexpr std::array unsupported_types { + FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT, + FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT, + FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE, + static_cast(-1), + static_cast(0), + static_cast(41), + static_cast(255), + }; + + for (const auto type : unsupported_types) { + SCOPED_TRACE(static_cast(type)); + EXPECT_THROW((void)storage_field_type_to_primitive_type(type), Exception); + } +} + +TEST(StorageFieldTypeTest, PersistedFieldTypeValuesStayStable) { + constexpr std::array persisted_types { + FieldType::OLAP_FIELD_TYPE_TINYINT, + FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT, + FieldType::OLAP_FIELD_TYPE_SMALLINT, + FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT, + FieldType::OLAP_FIELD_TYPE_INT, + FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, + FieldType::OLAP_FIELD_TYPE_BIGINT, + FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, + FieldType::OLAP_FIELD_TYPE_LARGEINT, + FieldType::OLAP_FIELD_TYPE_FLOAT, + FieldType::OLAP_FIELD_TYPE_DOUBLE, + FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE, + FieldType::OLAP_FIELD_TYPE_CHAR, + FieldType::OLAP_FIELD_TYPE_DATE, + FieldType::OLAP_FIELD_TYPE_DATETIME, + FieldType::OLAP_FIELD_TYPE_DECIMAL, + FieldType::OLAP_FIELD_TYPE_VARCHAR, + FieldType::OLAP_FIELD_TYPE_STRUCT, + FieldType::OLAP_FIELD_TYPE_ARRAY, + FieldType::OLAP_FIELD_TYPE_MAP, + FieldType::OLAP_FIELD_TYPE_UNKNOWN, + FieldType::OLAP_FIELD_TYPE_NONE, + FieldType::OLAP_FIELD_TYPE_HLL, + FieldType::OLAP_FIELD_TYPE_BOOL, + FieldType::OLAP_FIELD_TYPE_BITMAP, + FieldType::OLAP_FIELD_TYPE_STRING, + FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, + FieldType::OLAP_FIELD_TYPE_DATEV2, + FieldType::OLAP_FIELD_TYPE_DATETIMEV2, + FieldType::OLAP_FIELD_TYPE_TIMEV2, + FieldType::OLAP_FIELD_TYPE_DECIMAL32, + FieldType::OLAP_FIELD_TYPE_DECIMAL64, + FieldType::OLAP_FIELD_TYPE_DECIMAL128I, + FieldType::OLAP_FIELD_TYPE_JSONB, + FieldType::OLAP_FIELD_TYPE_VARIANT, + FieldType::OLAP_FIELD_TYPE_AGG_STATE, + FieldType::OLAP_FIELD_TYPE_DECIMAL256, + FieldType::OLAP_FIELD_TYPE_IPV4, + FieldType::OLAP_FIELD_TYPE_IPV6, + FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, + }; + + for (size_t i = 0; i < persisted_types.size(); ++i) { + EXPECT_EQ(static_cast(persisted_types[i]), i + 1); + } +} + +} // namespace + +} // namespace doris diff --git a/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp b/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp index 3769e5f69462d4..dbe9c8758348c3 100644 --- a/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp @@ -366,4 +366,33 @@ TEST_F(DataTypeDateTimeV2SerDeTest, ArrowMemNotAlignedDateTime) { EXPECT_TRUE(st.ok()); } +TEST_F(DataTypeDateTimeV2SerDeTest, ReadArrowTimestampBeforeEpoch) { + auto source_column = ColumnDateTimeV2::create(); + DateV2Value source_value; + source_value.unchecked_set_time(1969, 12, 31, 23, 59, 59, 123456); + source_column->insert_value(source_value); + std::string insert_value = "1969-12-31 23:59:59.123456"; + ASSERT_EQ(insert_value, source_value.to_string(6)); + + arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::MICRO), + arrow::default_memory_pool()); + ASSERT_TRUE(serde_datetime_v2_6 + ->write_column_to_arrow(*source_column, nullptr, &builder, 0, + source_column->size(), cctz::utc_time_zone()) + .ok()); + + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + const auto* timestamp_array = assert_cast(array.get()); + ASSERT_EQ(-876544, timestamp_array->Value(0)); + + auto dest_column = ColumnDateTimeV2::create(); + ASSERT_TRUE(serde_datetime_v2_6 + ->read_column_from_arrow(*dest_column, array.get(), 0, array->length(), + cctz::utc_time_zone()) + .ok()); + ASSERT_EQ(1, dest_column->size()); + EXPECT_EQ(insert_value, dest_column->get_element(0).to_string(6)); +} + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp index 69cf458e2fdc5f..18e4aa8a1cf848 100644 --- a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp @@ -365,7 +365,7 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadIntegersFromUnsignedSources) { TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalValues) { { - std::vector values = {0, 127, 255, 32767, 65535, -1}; + std::vector values = {0, 127, 255}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); auto result = read_column(std::make_shared(), view); @@ -375,12 +375,9 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalVal EXPECT_EQ(0, column.get_element(0)); EXPECT_EQ(127, column.get_element(1)); EXPECT_EQ(255, column.get_element(2)); - EXPECT_EQ(255, column.get_element(3)); - EXPECT_EQ(255, column.get_element(4)); - EXPECT_EQ(255, column.get_element(5)); } { - std::vector values = {32767, 65535, -1}; + std::vector values = {32767, 65535}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 16, false); auto result = read_column(std::make_shared(), view); @@ -389,7 +386,6 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalVal ASSERT_EQ(values.size(), column.size()); EXPECT_EQ(32767, column.get_element(0)); EXPECT_EQ(65535, column.get_element(1)); - EXPECT_EQ(65535, column.get_element(2)); } { std::vector values = {-1}; @@ -414,17 +410,42 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadUnsignedLogicalIntegersCastsPhysicalVal } } -TEST(DataTypeSerDeDecodedValuesTest, ReadSignedLogicalIntegersCastsPhysicalValues) { +TEST(DataTypeSerDeDecodedValuesTest, StrictSignedLogicalIntegersRejectOutOfRangePhysicalCarrier) { std::vector values = {127, 128, 255, -1}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, true); + // Carrier bounds are metadata validation: strict scans reject malformed values, while + // permissive scans intentionally preserve the declared bit-width conversion. + view.enable_strict_mode = true; auto result = read_column(std::make_shared(), view); + expect_data_quality_error(result.status); + EXPECT_EQ(result.column->size(), 0); +} + +TEST(DataTypeSerDeDecodedValuesTest, StrictUnsignedLogicalIntegersRejectOutOfRangePhysicalCarrier) { + std::vector values = {255, 256}; + auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); + view.enable_strict_mode = true; + auto result = read_column(std::make_shared(), view); + expect_data_quality_error(result.status); + EXPECT_EQ(result.column->size(), 0); +} + +TEST(DataTypeSerDeDecodedValuesTest, + NonStrictNullableLogicalIntegerCarrierPreservesBitWidthCompatibility) { + auto type = std::make_shared(std::make_shared()); + std::vector values = {127, 1000, -128}; + std::vector null_map(values.size(), 0); + auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values, &null_map), 8, + true); + auto result = read_column(type, view); + ASSERT_TRUE(result.status.ok()) << result.status; - const auto& column = assert_cast(*result.column); - ASSERT_EQ(values.size(), column.size()); - EXPECT_EQ(static_cast(127), column.get_element(0)); - EXPECT_EQ(static_cast(-128), column.get_element(1)); - EXPECT_EQ(static_cast(-1), column.get_element(2)); - EXPECT_EQ(static_cast(-1), column.get_element(3)); + const auto& nullable = assert_cast(*result.column); + const auto& nested = assert_cast(nullable.get_nested_column()); + EXPECT_FALSE(nullable.is_null_at(0)); + EXPECT_FALSE(nullable.is_null_at(1)); + EXPECT_FALSE(nullable.is_null_at(2)); + EXPECT_EQ(nested.get_data(), (ColumnInt8::Container {127, -24, -128})); } TEST(DataTypeSerDeDecodedValuesTest, ReadFloatAndDouble) { @@ -1633,7 +1654,7 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadFieldPrimitiveValues) { TEST(DataTypeSerDeDecodedValuesTest, ReadFieldLogicalIntegerCastsPhysicalValue) { { - std::vector values = {32767}; + std::vector values = {255}; auto view = with_logical_integer(make_fixed_view(DecodedValueKind::INT32, values), 8, false); auto field = read_field(std::make_shared(), view); diff --git a/be/test/core/data_type_serde/data_type_serde_number_test.cpp b/be/test/core/data_type_serde/data_type_serde_number_test.cpp index dc9099214b57df..b1050d7119b1af 100644 --- a/be/test/core/data_type_serde/data_type_serde_number_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_number_test.cpp @@ -29,6 +29,7 @@ #include #include "common/config.h" +#include "common/status.h" #include "core/assert_cast.h" #include "core/column/column.h" #include "core/data_type/common_data_type_serder_test.h" @@ -38,6 +39,7 @@ #include "core/data_type_serde/data_type_date_or_datetime_serde.h" #include "core/data_type_serde/data_type_datetimev2_serde.h" #include "core/data_type_serde/data_type_datev2_serde.h" +#include "core/field.h" #include "core/types.h" #include "testutil/test_util.h" #include "util/slice.h" @@ -618,4 +620,53 @@ TEST_F(DataTypeNumberSerDeTest, RowStoreDateTimeJsonbWidth) { }); } +// to_olap_string / from_zonemap_string must round-trip finite floating-point +// extremes (±DBL_MAX / ±FLT_MAX). The old digits10+1 (16g/7g) formatter rounded +// DBL_MAX up to 1.797693134862316e+308 — larger than the largest finite double — +// so from_olap_string parsed it to ±inf and rejected it, and the zone-map min/max +// never got materialized. +TEST_F(DataTypeNumberSerDeTest, OlapStringRoundTripFloatExtremes) { + auto check_double = [&](double v) { + Field field = Field::create_field(v); + std::string s = serde_float64->to_olap_string(field); + Field back; + Status st = serde_float64->from_zonemap_string(s, back); + ASSERT_TRUE(st.ok()) << "double round-trip failed: v=" << v << " str='" << s << "'"; + EXPECT_EQ(back.get(), v) << "str='" << s << "'"; + }; + auto check_float = [&](float v) { + Field field = Field::create_field(v); + std::string s = serde_float32->to_olap_string(field); + Field back; + Status st = serde_float32->from_zonemap_string(s, back); + ASSERT_TRUE(st.ok()) << "float round-trip failed: v=" << v << " str='" << s << "'"; + EXPECT_EQ(back.get(), v) << "str='" << s << "'"; + }; + + // finite extremes — these broke before the fix + check_double(std::numeric_limits::max()); // 1.7976931348623157e308 + check_double(std::numeric_limits::lowest()); // -1.7976931348623157e308 + check_float(std::numeric_limits::max()); + check_float(std::numeric_limits::lowest()); + + // neighbors of extremes + precision-sensitive values — also broke before + // the fix (digits10+1 is short of max_digits10) + check_double(std::nextafter(std::numeric_limits::max(), + 0.0)); // 1 ULP below DBL_MAX, 16g -> inf + check_double(std::numeric_limits::min()); // smallest normal, 16g loses precision + check_double(2.0000000164243876); // ordinary double needing 17 sig digits + check_float(std::nextafter(std::numeric_limits::max(), 0.0f)); // 1 ULP below FLT_MAX + check_float(std::numeric_limits::min()); // smallest normal float + + // ordinary values must still round-trip exactly + check_double(0.0); + check_double(-0.0); + check_double(1.0); + check_double(3.141592653589793); + check_double(1e300); + check_double(-1e308); + check_float(3.14f); + check_float(0.0f); +} + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp new file mode 100644 index 00000000000000..b7a4329f0d261b --- /dev/null +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -0,0 +1,891 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_decimal.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { +namespace { + +#pragma pack(1) +struct TestParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(TestParquetInt96Timestamp) == 12); + +constexpr int32_t JULIAN_UNIX_EPOCH = 2440588; +constexpr int64_t NANOS_PER_DAY = 86400000000000LL; + +class TestParquetDecodeSource final : public ParquetDecodeSource { +public: + template + void set_fixed_values(const std::vector& values) { + _value_width = sizeof(T); + _fixed_values.resize(values.size() * sizeof(T)); + memcpy(_fixed_values.data(), values.data(), _fixed_values.size()); + } + + void set_fixed_bytes(std::vector values, size_t value_width) { + _fixed_values = std::move(values); + _value_width = value_width; + } + + void set_dictionary(std::vector values, size_t value_width, + std::vector indices) { + _dictionary = std::move(values); + _dictionary_width = value_width; + _indices = std::move(indices); + _index_offset = 0; + ++_dictionary_generation; + } + + template + void set_fixed_dictionary(const std::vector& values, std::vector indices) { + std::vector encoded(values.size() * sizeof(T)); + memcpy(encoded.data(), values.data(), encoded.size()); + set_dictionary(std::move(encoded), sizeof(T), std::move(indices)); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_LE((_fixed_offset + num_values) * _value_width, _fixed_values.size()); + const uint8_t* begin = _fixed_values.data() + _fixed_offset * _value_width; + _fixed_offset += num_values; + return consumer.consume(begin, num_values, _value_width); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_LE(_binary_offset + num_values, _binary_refs.size()); + const StringRef* begin = _binary_refs.data() + _binary_offset; + _binary_offset += num_values; + return consumer.consume(begin, num_values); + } + + Status skip_values(size_t num_values) override { + _fixed_offset += num_values; + _binary_offset += num_values; + _index_offset += num_values; + return Status::OK(); + } + + bool has_dictionary() const override { return !_dictionary.empty(); } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + size_t dictionary_size() const override { + return _dictionary_width == 0 ? 0 : _dictionary.size() / _dictionary_width; + } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + ++_dictionary_decode_calls; + return fixed_consumer.consume(_dictionary.data(), dictionary_size(), _dictionary_width); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + DORIS_CHECK_LE(_index_offset + num_values, _indices.size()); + indices->assign(_indices.begin() + _index_offset, + _indices.begin() + _index_offset + num_values); + _index_offset += num_values; + return Status::OK(); + } + + size_t dictionary_decode_calls() const { return _dictionary_decode_calls; } + +private: + std::vector _fixed_values; + std::vector _binary_refs; + std::vector _dictionary; + std::vector _indices; + size_t _value_width = 0; + size_t _dictionary_width = 0; + size_t _fixed_offset = 0; + size_t _binary_offset = 0; + size_t _index_offset = 0; + uint64_t _dictionary_generation = 0; + size_t _dictionary_decode_calls = 0; +}; + +TEST(DataTypeSerDeParquetTest, MaterializesLogicalUnsignedIntegersDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 0, 7}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 32, + .logical_integer_is_signed = false}; + ParquetMaterializationState state; + DataTypeInt64 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 4294967295LL); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 7); +} + +TEST(DataTypeSerDeParquetTest, NonStrictNarrowUnsignedIntegersPreserveBitWidthCompatibility) { + TestParquetDecodeSource source; + source.set_fixed_values( + {0, 255, 32767, 65535, std::numeric_limits::max(), -1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 8, + .logical_integer_is_signed = false}; + IColumn::Filter null_map(6, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt16 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 6, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data, (ColumnInt16::Container {0, 255, 255, 255, 255, 255})); + EXPECT_EQ(null_map, IColumn::Filter(6, 0)); +} + +TEST(DataTypeSerDeParquetTest, LogicalIntegerRejectsOutOfRangePhysicalCarrier) { + TestParquetDecodeSource source; + source.set_fixed_values({1000}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 8, + .logical_integer_is_signed = true}; + ParquetMaterializationState state; + state.enable_strict_mode = true; + DataTypeInt8 type; + auto column = type.create_column(); + + EXPECT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .is()); + EXPECT_EQ(column->size(), 0); +} + +TEST(DataTypeSerDeParquetTest, NonStrictLogicalIntegerDictionaryPreservesBitWidthCompatibility) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 8, + .logical_integer_is_signed = true}; + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt8 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt8::Container {-24, -24})); + EXPECT_EQ(null_map, IColumn::Filter({0, 0})); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFloat16Directly) { + TestParquetDecodeSource source; + source.set_fixed_values({0x3C00, 0xC000, 0x7C00}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::FLOAT16, + .type_length = 2, + .logical_float16 = true}; + ParquetMaterializationState state; + DataTypeFloat32 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_FLOAT_EQ(data[0], 1.0F); + EXPECT_FLOAT_EQ(data[1], -2.0F); + EXPECT_TRUE(std::isinf(data[2])); +} + +TEST(DataTypeSerDeParquetTest, RescalesFixedBinaryDecimalDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x04, 0xD2, 0xFB, 0x2E}, 2); // 12.34 and -12.34 at scale 2. + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 2, + .decimal_precision = 4, + .decimal_scale = 2}; + ParquetMaterializationState state; + DataTypeDecimal64 type(18, 3); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 12340); + EXPECT_EQ(data[1].value, -12340); +} + +TEST(DataTypeSerDeParquetTest, RescalesExactIntegerDecimalsWithoutRowFailures) { + TestParquetDecodeSource source; + source.set_fixed_values({12300, -98700, 2147483600}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 10, + .decimal_scale = 4}; + ParquetMaterializationState state; + DataTypeDecimal64 type(9, 2); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + // Exact scale reduction can stay on the fast integer path without per-row failure bookkeeping. + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, -987); + EXPECT_EQ(data[2].value, 21474836); +} + +TEST(DataTypeSerDeParquetTest, DecimalScaleDownRejectsLossyPlainAndDictionaryValues) { + ParquetDecodeContext plain_context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 6, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + { + TestParquetDecodeSource source; + source.set_fixed_values({1230, 1234}); + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, 0); + EXPECT_EQ(null_map, IColumn::Filter({0, 1})); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({1234}); + ParquetMaterializationState state; + state.enable_strict_mode = true; + auto column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_dictionary({1230, 1234}, {1, 0, 1}); + auto dictionary_context = plain_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde() + ->read_column_from_parquet(*column, source, dictionary_context, 3, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[1].value, 123); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalUsesWideIntermediateBeforeNarrowing) { + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 19, + .decimal_scale = 0}; + DataTypeDecimal32 type(9, 0); + constexpr int64_t WRAPS_TO_ONE_IN_INT32 = 4294967297LL; + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_fixed_dictionary({WRAPS_TO_ONE_IN_INT32}, {0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_values({WRAPS_TO_ONE_IN_INT32}); + context.encoding = ParquetValueEncoding::PLAIN; + } + IColumn::Filter null_map(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1})); + EXPECT_EQ(assert_cast(*column).get_data()[0].value, 0); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalAcceptsSignExtendedBinaryAfterWideExactScaling) { + const std::vector values {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xCE, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFB, 0x32}; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 8, + .decimal_precision = 19, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_dictionary(values, 8, {1, 0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_bytes(values, 8); + context.encoding = ParquetValueEncoding::PLAIN; + } + ParquetMaterializationState state; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, dictionary ? -123 : 123); + EXPECT_EQ(data[1].value, dictionary ? 123 : -123); + } +} + +TEST(DataTypeSerDeParquetTest, ReusesTypedDictionary) { + TestParquetDecodeSource source; + std::vector dictionary {'a', 'a', 'b', 'b'}; + source.set_dictionary(std::move(dictionary), 2, {1, 0, 1, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2}; + ParquetMaterializationState state; + DataTypeString type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + ASSERT_EQ(state.typed_dictionary->size(), 2); + EXPECT_EQ(column->get_data_at(0).to_string(), "bb"); + EXPECT_EQ(column->get_data_at(1).to_string(), "aa"); + EXPECT_EQ(column->get_data_at(2).to_string(), "bb"); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 1); + EXPECT_EQ(column->get_data_at(3).to_string(), "aa"); + + source.set_dictionary({'c', 'c'}, 2, {0}); + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 2); + EXPECT_EQ(column->get_data_at(4).to_string(), "cc"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDictionaryIndicesWithoutValues) { + TestParquetDecodeSource source; + source.set_dictionary({'a', 'a', 'b', 'b'}, 2, {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2, + .dictionary_index_only = true}; + ParquetMaterializationState state; + DataTypeString type; + auto column = ColumnInt32::create(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = column->get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 1); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 1); + EXPECT_FALSE(state.typed_dictionary); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDateDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, 1, -1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ParquetMaterializationState state; + DataTypeDateV2 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 2), "1969-12-31"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampUnitsAndNegativeEpochDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, -1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1969-12-31 23:59:59.999999"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000001"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimeUnitsDirectly) { + struct TimeUnitCase { + ParquetTimeUnit unit; + int64_t units_per_day; + int64_t expected_last_micros; + }; + const std::vector cases { + {ParquetTimeUnit::MILLIS, 86400000, 86399999000}, + {ParquetTimeUnit::MICROS, 86400000000, 86399999999}, + {ParquetTimeUnit::NANOS, 86400000000000, 86399999999}, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(static_cast(test_case.unit)); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = test_case.unit}; + DataTypeTimeV2 type(6); + + TestParquetDecodeSource source; + source.set_fixed_values({0, test_case.units_per_day - 1}); + ParquetMaterializationState state; + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0], 0); + EXPECT_EQ(data[1], test_case.expected_last_micros); + + TestParquetDecodeSource nullable_source; + nullable_source.set_fixed_values({-1, test_case.units_per_day}); + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState nullable_state; + nullable_state.conversion_failure_null_map = &null_map; + auto nullable_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*nullable_column, nullable_source, context, + 2, nullable_state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + + TestParquetDecodeSource strict_source; + strict_source.set_fixed_values({-1}); + ParquetMaterializationState strict_state; + auto strict_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*strict_column, strict_source, context, 1, + strict_state) + .ok()); + EXPECT_EQ(strict_column->size(), 0); + + TestParquetDecodeSource unused_invalid_dictionary; + unused_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day - 1, test_case.units_per_day}, {1, 2}); + auto dictionary_context = context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState dictionary_state; + auto dictionary_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, + unused_invalid_dictionary, + dictionary_context, 2, dictionary_state) + .ok()); + EXPECT_EQ(dictionary_column->size(), 2); + + TestParquetDecodeSource referenced_invalid_dictionary; + referenced_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day}, {0}); + ParquetMaterializationState invalid_dictionary_state; + auto invalid_dictionary_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet( + *invalid_dictionary_column, referenced_invalid_dictionary, + dictionary_context, 1, invalid_dictionary_state) + .ok()); + EXPECT_EQ(invalid_dictionary_column->size(), 0); + } +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampTzDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1969); + EXPECT_EQ(data[0].microsecond(), 999999); + EXPECT_EQ(data[1].second(), 1); + EXPECT_EQ(data[1].microsecond(), 1); +} + +TEST(DataTypeSerDeParquetTest, ValidatesInt96BeforeDateTimeMaterialization) { + { + TestParquetDecodeSource source; + source.set_fixed_values( + {{0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY - 1, JULIAN_UNIX_EPOCH}}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 23:59:59.999999"); + } + + const std::vector invalid_values { + {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}, + {-1, JULIAN_UNIX_EPOCH}, + {0, std::numeric_limits::max()}}; + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, Int96DictionaryFailuresFollowDecodedIds) { + const std::vector dictionary_values { + {0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + std::vector dictionary(sizeof(TestParquetInt96Timestamp) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(TestParquetInt96Timestamp), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 00:00:00.000000"); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzChecksUnitOverflowAndTargetRange) { + constexpr int64_t MIN_TIMESTAMP_MICROS = -62135596800000000LL; + constexpr int64_t MAX_TIMESTAMP_MICROS = 253402300799999999LL; + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + + { + TestParquetDecodeSource source; + source.set_fixed_values({MIN_TIMESTAMP_MICROS, MAX_TIMESTAMP_MICROS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1); + EXPECT_EQ(data[1].year(), 9999); + EXPECT_EQ(data[1].microsecond(), 999999); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max(), YEAR_10000_MILLIS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDecodedValuesUseTheSameBounds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector values {std::numeric_limits::max(), YEAR_10000_MILLIS, 0}; + NullMap conversion_failures(3, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT64, + .time_unit = DecodedTimeUnit::MILLIS, + .row_count = 3, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(conversion_failures, NullMap({1, 1, 0})); + EXPECT_EQ(assert_cast(*column).get_data()[2].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDictionaryFailuresFollowDecodedIds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector dictionary_values {0, YEAR_10000_MILLIS}; + std::vector dictionary(sizeof(int64_t) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(int64_t), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(assert_cast(*column).get_data()[1].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, Int96DecodedValuesRejectInvalidNanos) { + const std::vector values {{NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + NullMap conversion_failures(1, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT96, + .row_count = 1, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(conversion_failures, NullMap({1})); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFixedBinaryAsVarbinaryDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F}, + 16); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::UUID, + .type_length = 16, + .logical_uuid = true}; + ParquetMaterializationState state; + DataTypeVarbinary type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->get_data_at(0).to_string(), std::string("\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F", + 16)); +} + +TEST(DataTypeSerDeParquetTest, NullableConversionFailuresBecomeNullOutsideStrictMode) { + auto expect_null = [](DataTypeSerDeSPtr serde, MutableColumnPtr column, + TestParquetDecodeSource* source, const ParquetDecodeContext& context) { + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + EXPECT_TRUE(serde->read_column_from_parquet(*column, *source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + }; + + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::min()}); + DataTypeDateV2 type; + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeDateTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({10}); + DataTypeDecimal32 type(1, 0); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 2, + .decimal_scale = 0}); + } +} + +TEST(DataTypeSerDeParquetTest, DictionaryConversionFailuresFollowDecodedIds) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt8 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map[0], 1); + EXPECT_EQ(null_map[1], 1); + EXPECT_EQ(source.dictionary_decode_calls(), 1); +} + +TEST(DataTypeSerDeParquetTest, NonNullableDictionaryConversionFailureRemainsAnError) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ParquetMaterializationState state; + DataTypeInt8 type; + auto column = type.create_column(); + + EXPECT_FALSE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 0); +} + +} // namespace +} // namespace doris diff --git a/be/test/core/jsonb/jsonb_document_cast_test.cpp b/be/test/core/jsonb/jsonb_document_cast_test.cpp index 7d3f4128d91f89..37ce572f744d1e 100644 --- a/be/test/core/jsonb/jsonb_document_cast_test.cpp +++ b/be/test/core/jsonb/jsonb_document_cast_test.cpp @@ -105,21 +105,21 @@ class JsonbDocumentCastTest : public testing::Test { { JsonbWriter writer; - Decimal64 dec = 1234567890123456789; + Decimal64 dec = int64_t(1234567890123456789); writer.writeDecimal(dec, 18, 2); to_jsonbs(writer); } { JsonbWriter writer; - Decimal128V3 dec = 1234567890123456789; + Decimal128V3 dec = int64_t(1234567890123456789); writer.writeDecimal(dec, 38, 3); to_jsonbs(writer); } { JsonbWriter writer; - Decimal256 dec {1234567890123456789}; + Decimal256 dec {int64_t(1234567890123456789)}; writer.writeDecimal(dec, 76, 3); to_jsonbs(writer); } diff --git a/be/test/core/value/bitmap_value_test.cpp b/be/test/core/value/bitmap_value_test.cpp index 1f89539c366cc7..b281d71a443433 100644 --- a/be/test/core/value/bitmap_value_test.cpp +++ b/be/test/core/value/bitmap_value_test.cpp @@ -78,6 +78,27 @@ TEST(BitmapValueTest, Roaring64Map_ctors) { EXPECT_FALSE(roaring64_map3.contains(uint32_t(0))); } +TEST(BitmapValueTest, Roaring64Map_intersect) { + constexpr uint64_t high32 = uint64_t {1} << 32; + const std::vector one_map {1, 2}; + const std::vector one_map_overlap {2, 3}; + const std::vector one_map_disjoint {3, 4}; + const std::vector other_map {high32 + 1, high32 + 2}; + const std::vector two_maps {1, 2, high32 + 1, high32 + 2}; + + detail::Roaring64Map bitmap_one_map(one_map.size(), one_map.data()); + detail::Roaring64Map bitmap_overlap(one_map_overlap.size(), one_map_overlap.data()); + detail::Roaring64Map bitmap_disjoint(one_map_disjoint.size(), one_map_disjoint.data()); + detail::Roaring64Map bitmap_other_map(other_map.size(), other_map.data()); + detail::Roaring64Map bitmap_two_maps(two_maps.size(), two_maps.data()); + + EXPECT_TRUE(bitmap_one_map.intersect(bitmap_overlap)); + EXPECT_FALSE(bitmap_one_map.intersect(bitmap_disjoint)); + EXPECT_FALSE(bitmap_one_map.intersect(bitmap_other_map)); + EXPECT_TRUE(bitmap_two_maps.intersect(bitmap_one_map)); + EXPECT_TRUE(bitmap_one_map.intersect(bitmap_two_maps)); +} + TEST(BitmapValueTest, Roaring64Map_add_remove) { detail::Roaring64Map roaring64_map; const std::vector values({1, 3, 5, 7, 9, 2, 4, 6, 8, 1, 8, 9}); @@ -252,17 +273,6 @@ TEST(BitmapValueTest, Roaring64Map_iterators) { EXPECT_TRUE(iter != end); iter++; } - - iter = roaring64_map.begin(); - EXPECT_TRUE(iter.move(2)); - EXPECT_TRUE(iter.move(4294967296)); - EXPECT_FALSE(iter.move(4294967299)); - - iter = roaring64_map.begin(); - auto iter2 = roaring64_map.begin(); - iter.move(3); - iter2.move(3); - EXPECT_TRUE(iter == iter2); } TEST(BitmapValueTest, set) { @@ -359,6 +369,80 @@ TEST(BitmapValueTest, add) { config::enable_set_in_bitmap_value = false; } +TEST(BitmapValueTest, small_set_try_insert_many_over_capacity) { + BitmapSmallSet set; + const uint64_t initial_values[] = {1, 2, 3}; + const uint64_t duplicate_values[] = {1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, + 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3}; + const uint64_t new_values[] = {4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33}; + + EXPECT_TRUE(set.try_insert_many(initial_values, std::size(initial_values))); + EXPECT_TRUE(set.try_insert_many(duplicate_values, std::size(duplicate_values))); + EXPECT_EQ(3, set.size()); + EXPECT_FALSE(set.try_insert_many(new_values, std::size(new_values))); + EXPECT_EQ(3, set.size()); +} + +TEST(BitmapValueTest, add_many_all_representations) { + const auto old_enable_set = config::enable_set_in_bitmap_value; + config::enable_set_in_bitmap_value = true; + + const uint64_t one_value[] = {1}; + const uint64_t small_values[] = {1, 2, 2, 3}; + std::vector bitmap_values(33); + std::iota(bitmap_values.begin(), bitmap_values.end(), 1); + + BitmapValue empty; + empty.add_many(one_value, 0); + EXPECT_TRUE(empty.empty()); + empty.add_many(one_value, std::size(one_value)); + EXPECT_EQ(BitmapValue::SINGLE, empty._type); + + BitmapValue empty_to_set; + empty_to_set.add_many(small_values, std::size(small_values)); + EXPECT_EQ(BitmapValue::SET, empty_to_set._type); + EXPECT_EQ(3, empty_to_set.cardinality()); + + BitmapValue empty_to_bitmap; + empty_to_bitmap.add_many(bitmap_values.data(), bitmap_values.size()); + EXPECT_EQ(BitmapValue::BITMAP, empty_to_bitmap._type); + + BitmapValue single_to_set(1); + single_to_set.add_many(small_values, std::size(small_values)); + EXPECT_EQ(BitmapValue::SET, single_to_set._type); + EXPECT_EQ(3, single_to_set.cardinality()); + + BitmapValue single_to_bitmap(uint64_t {0}); + single_to_bitmap.add_many(bitmap_values.data(), bitmap_values.size()); + EXPECT_EQ(BitmapValue::BITMAP, single_to_bitmap._type); + EXPECT_EQ(34, single_to_bitmap.cardinality()); + + empty_to_bitmap.add_many(one_value, std::size(one_value)); + EXPECT_EQ(33, empty_to_bitmap.cardinality()); + + std::vector set_values(31); + std::iota(set_values.begin(), set_values.end(), 0); + BitmapValue set_to_bitmap(set_values); + const uint64_t duplicates[] = {0, 1}; + const uint64_t overflow_values[] = {31, 33}; + set_to_bitmap.add_many(duplicates, std::size(duplicates)); + EXPECT_EQ(BitmapValue::SET, set_to_bitmap._type); + set_to_bitmap.add_many(overflow_values, std::size(overflow_values)); + EXPECT_EQ(BitmapValue::BITMAP, set_to_bitmap._type); + EXPECT_EQ(33, set_to_bitmap.cardinality()); + + config::enable_set_in_bitmap_value = false; + BitmapValue no_set_empty; + no_set_empty.add_many(small_values, std::size(small_values)); + EXPECT_EQ(BitmapValue::BITMAP, no_set_empty._type); + BitmapValue no_set_single(uint64_t {0}); + no_set_single.add_many(one_value, std::size(one_value)); + EXPECT_EQ(BitmapValue::BITMAP, no_set_single._type); + + config::enable_set_in_bitmap_value = old_enable_set; +} + void check_bitmap_value_operator(const BitmapValue& left, const BitmapValue& right) { auto left_cardinality = left.cardinality(); auto right_cardinality = right.cardinality(); @@ -492,6 +576,88 @@ TEST(BitmapValueTest, operators) { config::enable_set_in_bitmap_value = false; } +TEST(BitmapValueTest, bitmap_operator_cow_with_set_rhs) { + config::enable_set_in_bitmap_value = true; + + std::vector values(128); + std::iota(values.begin(), values.end(), 0); + + BitmapValue bitmap; + bitmap.add_many(values.data(), values.size()); + + BitmapValue rhs_set({1, 2, 200}); + + BitmapValue copied = bitmap; + bitmap -= rhs_set; + EXPECT_FALSE(bitmap.contains(1)); + EXPECT_FALSE(bitmap.contains(2)); + EXPECT_TRUE(copied.contains(1)); + EXPECT_TRUE(copied.contains(2)); + + bitmap = copied; + copied = bitmap; + bitmap |= rhs_set; + EXPECT_TRUE(copied.contains(1)); + EXPECT_TRUE(copied.contains(2)); + EXPECT_FALSE(copied.contains(200)); + + bitmap = copied; + copied = bitmap; + bitmap ^= BitmapValue(1); + EXPECT_FALSE(bitmap.contains(1)); + EXPECT_TRUE(copied.contains(1)); + + config::enable_set_in_bitmap_value = false; +} + +TEST(BitmapValueTest, bitmap_xor_single_and_set) { + config::enable_set_in_bitmap_value = true; + + BitmapValue lhs_single(1); + lhs_single ^= BitmapValue(2); + EXPECT_STREQ("1,2", lhs_single.to_string().c_str()); + + lhs_single = BitmapValue(1); + lhs_single ^= BitmapValue({2, 3}); + EXPECT_STREQ("1,2,3", lhs_single.to_string().c_str()); + + lhs_single = BitmapValue(1); + lhs_single ^= BitmapValue({1, 2}); + EXPECT_STREQ("2", lhs_single.to_string().c_str()); + + config::enable_set_in_bitmap_value = false; +} + +TEST(BitmapValueTest, bitmap_xor_same_set_returns_empty) { + config::enable_set_in_bitmap_value = true; + + BitmapValue lhs({1, 2}); + lhs ^= BitmapValue({1, 2}); + + EXPECT_EQ(0, lhs.cardinality()); + EXPECT_EQ(BitmapTypeCode::EMPTY, lhs.get_type_code()); + + config::enable_set_in_bitmap_value = false; +} + +TEST(BitmapValueTest, bitmap_xor_set_bitmap_clear_stale_set) { + config::enable_set_in_bitmap_value = true; + + BitmapValue lhs({1, 2}); + std::vector rhs_values(33); + std::iota(rhs_values.begin(), rhs_values.end(), 1); + BitmapValue rhs(rhs_values); + + lhs ^= rhs; + for (uint64_t v = 3; v <= 32; ++v) { + lhs.remove(v); + } + + EXPECT_STREQ("33", lhs.to_string().c_str()); + + config::enable_set_in_bitmap_value = false; +} + void check_bitmap_equal(const BitmapValue& left, const BitmapValue& right) { EXPECT_EQ(left.cardinality(), right.cardinality()); EXPECT_EQ(left.minimum(), right.minimum()); @@ -721,6 +887,18 @@ TEST(BitmapValueTest, sub_range_limit) { config::enable_set_in_bitmap_value = false; } +TEST(BitmapValueTest, offset_limit_negative_cardinality_for_set) { + config::enable_set_in_bitmap_value = true; + + BitmapValue bitmap({1, 2, 3}); + BitmapValue out; + auto ret = bitmap.offset_limit(-3, 2, &out); + EXPECT_EQ(ret, 2); + EXPECT_STREQ("1,2", out.to_string().c_str()); + + config::enable_set_in_bitmap_value = false; +} + void bitmap_checker_for_all_type(const std::function& checker) { BitmapValue bitmap_empty; BitmapValue bitmap_single(1); @@ -913,6 +1091,24 @@ TEST(BitmapValueTest, bitmap_union) { config::enable_set_in_bitmap_value = old_config; } +TEST(BitmapValueTest, fastunion_set_with_bitmap) { + const auto old_enable_set = config::enable_set_in_bitmap_value; + config::enable_set_in_bitmap_value = true; + + std::vector bitmap_values(33); + std::iota(bitmap_values.begin(), bitmap_values.end(), 3); + BitmapValue bitmap(bitmap_values); + BitmapValue set({1, 2}); + + set.fastunion({&bitmap}); + EXPECT_EQ(BitmapValue::BITMAP, set._type); + EXPECT_EQ(35, set.cardinality()); + EXPECT_TRUE(set.contains(1)); + EXPECT_TRUE(set.contains(35)); + + config::enable_set_in_bitmap_value = old_enable_set; +} + TEST(BitmapValueTest, bitmap_intersect) { BitmapValue empty; BitmapValue single(1024); @@ -976,6 +1172,51 @@ TEST(BitmapValueTest, bitmap_intersect) { EXPECT_EQ(2, bitmap6.cardinality()); } +TEST(BitmapValueTest, bitmap_intersects_all_representations) { + const auto old_enable_set = config::enable_set_in_bitmap_value; + + config::enable_set_in_bitmap_value = false; + std::vector bitmap_values(33); + std::iota(bitmap_values.begin(), bitmap_values.end(), 0); + std::vector bitmap_overlap_values(33); + std::iota(bitmap_overlap_values.begin(), bitmap_overlap_values.end(), 32); + std::vector bitmap_disjoint_values(33); + std::iota(bitmap_disjoint_values.begin(), bitmap_disjoint_values.end(), 100); + BitmapValue bitmap(bitmap_values); + BitmapValue bitmap_overlap(bitmap_overlap_values); + BitmapValue bitmap_disjoint(bitmap_disjoint_values); + BitmapValue single(1); + BitmapValue single_absent(200); + BitmapValue empty; + + config::enable_set_in_bitmap_value = true; + BitmapValue set({1, 2, 3}); + BitmapValue set_disjoint({4, 5}); + + EXPECT_FALSE(bitmap.intersects(empty)); + EXPECT_FALSE(empty.intersects(single)); + EXPECT_TRUE(bitmap.intersects(single)); + EXPECT_TRUE(set.intersects(single)); + + EXPECT_FALSE(empty.intersects(bitmap)); + EXPECT_TRUE(single.intersects(bitmap)); + EXPECT_FALSE(single_absent.intersects(bitmap)); + EXPECT_TRUE(bitmap.intersects(bitmap_overlap)); + EXPECT_FALSE(bitmap.intersects(bitmap_disjoint)); + EXPECT_TRUE(set.intersects(bitmap)); + EXPECT_FALSE(set_disjoint.intersects(bitmap_disjoint)); + + EXPECT_FALSE(empty.intersects(set)); + EXPECT_TRUE(single.intersects(set)); + EXPECT_FALSE(single_absent.intersects(set)); + EXPECT_TRUE(bitmap.intersects(set)); + EXPECT_FALSE(bitmap_disjoint.intersects(set)); + EXPECT_TRUE(set.intersects(BitmapValue({2, 4}))); + EXPECT_FALSE(set.intersects(set_disjoint)); + + config::enable_set_in_bitmap_value = old_enable_set; +} + std::string convert_bitmap_to_string(BitmapValue& bitmap) { std::string buf; buf.resize(bitmap.getSizeInBytes()); @@ -995,6 +1236,36 @@ BitmapValue deserialize_bitmap_from_string(const std::string& buffer) { return bitmap; } +TEST(BitmapValueTest, deserialize_reused_bitmap_to_set_then_promote) { + const auto old_enable_set = config::enable_set_in_bitmap_value; + config::enable_set_in_bitmap_value = true; + + std::vector stale_values(41); + std::iota(stale_values.begin(), stale_values.end(), 0); + BitmapValue reused(stale_values); + EXPECT_EQ(BitmapValue::BITMAP, reused._type); + + std::vector set_values {100, 101}; + BitmapValue set_bitmap(set_values); + EXPECT_EQ(BitmapValue::SET, set_bitmap._type); + std::string set_buffer = convert_bitmap_to_string(set_bitmap); + + EXPECT_TRUE(reused.deserialize(set_buffer.data())); + EXPECT_EQ(BitmapValue::SET, reused._type); + check_bitmap_values(reused, set_values); + + std::vector promote_values(31); + std::iota(promote_values.begin(), promote_values.end(), 102); + reused.add_many(promote_values.data(), promote_values.size()); + + std::vector expected {100, 101}; + expected.insert(expected.end(), promote_values.begin(), promote_values.end()); + EXPECT_EQ(BitmapValue::BITMAP, reused._type); + check_bitmap_values(reused, expected); + + config::enable_set_in_bitmap_value = old_enable_set; +} + TEST(BitmapValueTest, bitmap_serde) { auto old_enable_set = config::enable_set_in_bitmap_value; auto old_serialize_version = config::bitmap_serialize_version; @@ -1293,6 +1564,162 @@ TEST(BitmapValueTest, bitmap_value_iterator_test) { } } +TEST(BitmapValueTest, contains_all_ignore_internal_representation) { + bool old_enable_set = config::enable_set_in_bitmap_value; + + config::enable_set_in_bitmap_value = false; + BitmapValue lhs_bitmap({1, 2, 3}); + BitmapValue rhs_bitmap({1, 2}); + EXPECT_EQ(BitmapValue::BITMAP, lhs_bitmap._type); + EXPECT_EQ(BitmapValue::BITMAP, rhs_bitmap._type); + + config::enable_set_in_bitmap_value = true; + BitmapValue lhs_set({1, 2, 3}); + BitmapValue rhs_set({1, 2}); + EXPECT_EQ(BitmapValue::SET, lhs_set._type); + EXPECT_EQ(BitmapValue::SET, rhs_set._type); + + EXPECT_TRUE(lhs_set.contains_all(rhs_bitmap)); + EXPECT_TRUE(lhs_bitmap.contains_all(rhs_set)); + EXPECT_TRUE(lhs_bitmap.contains_all(rhs_bitmap)); + EXPECT_FALSE(rhs_set.contains_all(lhs_bitmap)); + + BitmapValue empty_set_rhs({1, 3}); + empty_set_rhs &= BitmapValue({2, 4}); + EXPECT_EQ(0, empty_set_rhs.cardinality()); + EXPECT_TRUE(lhs_set.contains_all(empty_set_rhs)); + EXPECT_TRUE(BitmapValue().contains_all(empty_set_rhs)); + + config::enable_set_in_bitmap_value = old_enable_set; +} + +TEST(BitmapValueTest, contains_all_representation_branches) { + const auto old_enable_set = config::enable_set_in_bitmap_value; + + config::enable_set_in_bitmap_value = false; + BitmapValue bitmap({1, 2, 3}); + const uint64_t duplicated_one[] = {1, 1}; + BitmapValue bitmap_one; + bitmap_one.add_many(duplicated_one, std::size(duplicated_one)); + BitmapValue bitmap_two({1, 2}); + BitmapValue bitmap_missing({1, 4}); + BitmapValue single(1); + + config::enable_set_in_bitmap_value = true; + BitmapValue set({1, 2, 3}); + BitmapValue set_one; + set_one.add(1); + BitmapValue set_missing({1, 4}); + + EXPECT_FALSE(BitmapValue().contains_all(bitmap_two)); + EXPECT_TRUE(single.contains_all(BitmapValue(1))); + EXPECT_FALSE(single.contains_all(BitmapValue(2))); + EXPECT_TRUE(single.contains_all(bitmap_one)); + EXPECT_FALSE(single.contains_all(bitmap_two)); + EXPECT_TRUE(set.contains_all(bitmap_two)); + EXPECT_FALSE(set.contains_all(bitmap_missing)); + EXPECT_TRUE(bitmap.contains_all(set_one)); + EXPECT_FALSE(bitmap.contains_all(set_missing)); + EXPECT_TRUE(single.contains_all(set_one)); + EXPECT_FALSE(single.contains_all(set_missing)); + + config::enable_set_in_bitmap_value = old_enable_set; +} + +TEST(BitmapValueTest, deserialize_set_duplicate_values) { + bool old_enable_set = config::enable_set_in_bitmap_value; + config::enable_set_in_bitmap_value = true; + + { + char data[] = {static_cast(BitmapTypeCode::SET), + 2, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0}; + BitmapValue bitmap; + EXPECT_THROW(bitmap.deserialize(data), Exception); + } + + { + char data[] = {static_cast(BitmapTypeCode::SET_V2), + 2, + 0, + 0, + 0, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0}; + BitmapValue bitmap; + EXPECT_TRUE(bitmap.deserialize(data)); + EXPECT_EQ(BitmapValue::SET, bitmap._type); + EXPECT_EQ(1, bitmap.cardinality()); + EXPECT_TRUE(bitmap.contains(7)); + } + + config::enable_set_in_bitmap_value = old_enable_set; +} + +TEST(BitmapValueTest, deserialize_set_v2_to_bitmap) { + const auto old_enable_set = config::enable_set_in_bitmap_value; + config::enable_set_in_bitmap_value = false; + + char data[] = {static_cast(BitmapTypeCode::SET_V2), + 2, + 0, + 0, + 0, + 7, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0}; + BitmapValue bitmap; + EXPECT_TRUE(bitmap.deserialize(data)); + EXPECT_EQ(BitmapValue::BITMAP, bitmap._type); + EXPECT_EQ(2, bitmap.cardinality()); + EXPECT_TRUE(bitmap.contains(7)); + EXPECT_TRUE(bitmap.contains(9)); + + config::enable_set_in_bitmap_value = old_enable_set; +} + TEST(BitmapValueTest, invalid_data) { BitmapValue bitmap; char data[] = {0x02, static_cast(0xff), 0x03}; diff --git a/be/test/exec/column_type_convert_test.cpp b/be/test/exec/column_type_convert_test.cpp index 2b88ea9228ef76..67e321c21d0f35 100644 --- a/be/test/exec/column_type_convert_test.cpp +++ b/be/test/exec/column_type_convert_test.cpp @@ -373,8 +373,8 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { auto& src_data = src_col->get_data(); // Add test values - src_data.push_back(Decimal64(12345678901234)); // Normal value: 1234567890.1234 - src_data.push_back(Decimal64(-98765432109876)); // Negative value: -9876543210.9876 + src_data.push_back(Decimal64(int64_t(12345678901234))); // Normal value: 1234567890.1234 + src_data.push_back(Decimal64(int64_t(-98765432109876))); // Negative value: -9876543210.9876 auto mutable_dst = dst_type->create_column(); @@ -485,7 +485,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(Decimal256(327673345)); src_data.push_back(Decimal256(655353345)); src_data.push_back(Decimal256(655363345)); - src_data.push_back(Decimal256(3333333333332345)); + src_data.push_back(Decimal256(int64_t(3333333333332345))); auto mutable_dst = nullable_dst_type->create_column(); auto& nullable_col = static_cast(*mutable_dst); diff --git a/be/test/exec/operator/query_cache_operator_test.cpp b/be/test/exec/operator/query_cache_operator_test.cpp index 91c73b99077247..177e1aef7f96ee 100644 --- a/be/test/exec/operator/query_cache_operator_test.cpp +++ b/be/test/exec/operator/query_cache_operator_test.cpp @@ -65,11 +65,18 @@ struct QueryCacheOperatorTest : public ::testing::Test { scan_ranges.push_back(scan_range); } void TearDown() override { - // Must clear state before query_cache_uptr is destroyed, because - // local states inside `state` hold QueryCacheHandle which references - // the QueryCache. C++ destroys members in reverse declaration order, - // so state (declared before query_cache_uptr) would outlive the cache. + // Must release every QueryCacheHandle holder before query_cache_uptr is + // destroyed. C++ destroys members in reverse declaration order, so + // `state` (local states) and `source`/`sink` (whose QueryCacheRuntime + // owns the decisions that pin cache entries) would otherwise outlive + // the cache and release their handles into a destroyed QueryCache. + // Local states built manually and never moved into `state` are + // released here too, for the same reason. + source_local_state_uptr.reset(); + sink_local_state_uptr.reset(); state.reset(); + source.reset(); + sink.reset(); } void create_local_state() { shared_state = sink->create_shared_state(); @@ -134,10 +141,6 @@ struct QueryCacheOperatorTest : public ::testing::Test { TEST_F(QueryCacheOperatorTest, test_no_hit_cache1) { sink = std::make_unique(); - source = std::make_unique(); - EXPECT_TRUE(source->set_child(child_op)); - child_op->_mock_row_desc.reset( - new MockRowDescriptor {{std::make_shared()}, &pool}); TQueryCacheParam cache_param; cache_param.node_id = 0; cache_param.digest = "test_digest"; @@ -147,7 +150,14 @@ TEST_F(QueryCacheOperatorTest, test_no_hit_cache1) { cache_param.entry_max_bytes = 1024 * 1024; cache_param.entry_max_rows = 1000; - source->_cache_param = cache_param; + // Exercise the production constructor once; the other tests use the + // BE_TEST default constructor and assign the members directly. + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); create_local_state(); std::cout << query_cache->get_element_count() << std::endl; @@ -170,7 +180,11 @@ TEST_F(QueryCacheOperatorTest, test_no_hit_cache1) { block, ColumnHelper::create_block({1, 2, 3, 4, 5}))); EXPECT_EQ(query_cache->get_element_count(), 1); } - EXPECT_TRUE(source_local_state->_need_insert_cache); + // A successful commit latches the flag off (the insert is exactly-once); + // that the query cached at all is asserted by the element count above, + // in contrast to test_no_hit_cache2 where the over-limit entry is + // dropped and the count stays 0. + EXPECT_FALSE(source_local_state->_need_insert_cache); } TEST_F(QueryCacheOperatorTest, test_no_hit_cache2) { @@ -189,6 +203,7 @@ TEST_F(QueryCacheOperatorTest, test_no_hit_cache2) { cache_param.entry_max_rows = 3; source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); create_local_state(); std::cout << query_cache->get_element_count() << std::endl; @@ -241,6 +256,7 @@ TEST_F(QueryCacheOperatorTest, test_hit_cache) { } source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); create_local_state(); std::cout << query_cache->get_element_count() << std::endl; @@ -276,4 +292,728 @@ TEST_F(QueryCacheOperatorTest, test_hit_cache) { } } +TEST_F(QueryCacheOperatorTest, test_stale_full_recompute) { + sink = std::make_unique(); + source = std::make_unique(); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + + std::string cache_key; + { + // A stale entry (version 100 < scan version 114514) without + // allow_incremental: plain miss, full recompute, write back overwrites. + int64_t version = 0; + EXPECT_TRUE( + QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + CacheResult result; + result.push_back(std::make_unique()); + *result.back() = ColumnHelper::create_block({100, 200}); + query_cache->insert(cache_key, 100, result, {0}, 1); + } + + source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); + create_local_state(); + + EXPECT_TRUE(source_local_state->_need_insert_cache); + EXPECT_FALSE(source_local_state->_is_incremental); + + { + auto block = ColumnHelper::create_block({1, 2, 3, 4, 5}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + { + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 5); + EXPECT_TRUE(ColumnHelper::block_equal( + block, ColumnHelper::create_block({1, 2, 3, 4, 5}))); + } + + // The stale entry is overwritten by the recomputed one. + doris::QueryCacheHandle handle; + EXPECT_TRUE(query_cache->lookup(cache_key, 114514, &handle)); + EXPECT_EQ(handle.get_cache_delta_count(), 0); +} + +TEST_F(QueryCacheOperatorTest, test_incremental_merge) { + sink = std::make_unique(); + source = std::make_unique(); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + cache_param.__set_allow_incremental(true); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + { + // The cached partial blocks of version 100, already merged once before. + CacheResult result; + result.push_back(std::make_unique()); + *result.back() = ColumnHelper::create_block({1, 2, 3, 4, 5}); + query_cache->insert(cache_key, 100, result, {0}, 1, /*delta_count=*/1); + } + + source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); + // Hand-craft the INCREMENTAL decision (capturing a real delta read source + // needs a storage engine, which unit tests do not have). The scan side + // would scan only (100, 114514] and feed the delta through the sink. + { + auto decision = std::make_shared(); + decision->mode = QueryCacheInstanceDecision::Mode::INCREMENTAL; + decision->key_valid = true; + decision->cache_key = cache_key; + decision->current_version = version; + decision->cached_version = 100; + decision->cached_delta_count = 1; + EXPECT_TRUE(query_cache->lookup_any_version(cache_key, &decision->handle)); + source->_query_cache_runtime->inject_decision_for_test(cache_key, decision); + } + create_local_state(); + + EXPECT_TRUE(source_local_state->_need_insert_cache); + EXPECT_TRUE(source_local_state->_is_incremental); + const std::string* stale = + source_local_state->custom_profile()->get_info_string("HitCacheStale"); + ASSERT_NE(stale, nullptr); + EXPECT_EQ(*stale, "1"); + const std::string* delta_versions = + source_local_state->custom_profile()->get_info_string("IncrementalDeltaVersions"); + ASSERT_NE(delta_versions, nullptr); + EXPECT_EQ(*delta_versions, "(100, 114514]"); + + { + // The delta partial result produced by scanning only (100, 114514]. + auto block = ColumnHelper::create_block({6, 7}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + { + // First the cached blocks are emitted ... + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 5); + EXPECT_TRUE(ColumnHelper::block_equal( + block, ColumnHelper::create_block({1, 2, 3, 4, 5}))); + } + { + // ... then the delta from the data queue; both are partial aggregation + // states merged by the upstream aggregation. + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 2); + EXPECT_TRUE(ColumnHelper::block_equal(block, + ColumnHelper::create_block({6, 7}))); + } + + // The merged entry is written back under the new version with an increased + // delta count, holding both the cached and the delta blocks. + doris::QueryCacheHandle handle; + EXPECT_TRUE(query_cache->lookup(cache_key, 114514, &handle)); + EXPECT_EQ(handle.get_cache_delta_count(), 2); + int64_t total_rows = 0; + for (const auto& cached_block : *handle.get_cache_result()) { + total_rows += cached_block->rows(); + } + EXPECT_EQ(total_rows, 7); +} + +TEST_F(QueryCacheOperatorTest, test_incremental_over_entry_limit) { + sink = std::make_unique(); + source = std::make_unique(); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + // The cached part alone (5 rows) already exceeds the limit, so the merged + // entry must not be written back; the stale entry stays untouched. + cache_param.entry_max_rows = 3; + cache_param.__set_allow_incremental(true); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + { + CacheResult result; + result.push_back(std::make_unique()); + *result.back() = ColumnHelper::create_block({1, 2, 3, 4, 5}); + query_cache->insert(cache_key, 100, result, {0}, 1, /*delta_count=*/0); + } + + source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); + { + auto decision = std::make_shared(); + decision->mode = QueryCacheInstanceDecision::Mode::INCREMENTAL; + decision->key_valid = true; + decision->cache_key = cache_key; + decision->current_version = version; + decision->cached_version = 100; + decision->cached_delta_count = 0; + EXPECT_TRUE(query_cache->lookup_any_version(cache_key, &decision->handle)); + source->_query_cache_runtime->inject_decision_for_test(cache_key, decision); + } + create_local_state(); + + EXPECT_TRUE(source_local_state->_need_insert_cache); + + { + auto block = ColumnHelper::create_block({6, 7}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + { + // Emitting the cached blocks overruns entry_max_rows: caching stops but + // the data still flows. + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 5); + EXPECT_FALSE(source_local_state->_need_insert_cache); + } + { + // The delta passes through unchanged. + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 2); + EXPECT_TRUE(ColumnHelper::block_equal(block, + ColumnHelper::create_block({6, 7}))); + } + + // No write back happened: the stale entry still carries version 100. + doris::QueryCacheHandle handle; + EXPECT_FALSE(query_cache->lookup(cache_key, 114514, &handle)); + doris::QueryCacheHandle stale_handle; + EXPECT_TRUE(query_cache->lookup_any_version(cache_key, &stale_handle)); + EXPECT_EQ(stale_handle.get_cache_version(), 100); +} + +TEST_F(QueryCacheOperatorTest, test_incremental_write_back_infeasible) { + sink = std::make_unique(); + source = std::make_unique(); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + cache_param.__set_allow_incremental(true); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + { + CacheResult result; + result.push_back(std::make_unique()); + *result.back() = ColumnHelper::create_block({1, 2, 3, 4, 5}); + query_cache->insert(cache_key, 100, result, {0}, 1); + } + + source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); + // The decision already knows the merged entry could never fit the limits: + // scan only the delta, but never clone blocks for a doomed write back. + { + auto decision = std::make_shared(); + decision->mode = QueryCacheInstanceDecision::Mode::INCREMENTAL; + decision->key_valid = true; + decision->write_back_feasible = false; + decision->cache_key = cache_key; + decision->current_version = version; + decision->cached_version = 100; + decision->cached_delta_count = 0; + EXPECT_TRUE(query_cache->lookup_any_version(cache_key, &decision->handle)); + source->_query_cache_runtime->inject_decision_for_test(cache_key, decision); + } + create_local_state(); + + EXPECT_FALSE(source_local_state->_need_insert_cache); + EXPECT_TRUE(source_local_state->_is_incremental); + + { + auto block = ColumnHelper::create_block({6, 7}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + { + // Cached blocks flow through without being cloned for a write back. + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 5); + EXPECT_TRUE(source_local_state->_local_cache_blocks.empty()); + } + { + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 2); + EXPECT_TRUE(source_local_state->_local_cache_blocks.empty()); + } + + // No write back: the stale entry still carries version 100. + doris::QueryCacheHandle handle; + EXPECT_FALSE(query_cache->lookup(cache_key, 114514, &handle)); + doris::QueryCacheHandle stale_handle; + EXPECT_TRUE(query_cache->lookup_any_version(cache_key, &stale_handle)); + EXPECT_EQ(stale_handle.get_cache_version(), 100); +} + +TEST_F(QueryCacheOperatorTest, test_incremental_fallback_reason_in_profile) { + sink = std::make_unique(); + source = std::make_unique(); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + // Use a tablet id that exists nowhere so fetching the tablet fails even if + // another suite left a storage engine registered in this test process. + constexpr int64_t kMissingTabletId = 424242424242; + scan_ranges.clear(); + TScanRangeParams scan_range; + TPaloScanRange palo_scan_range; + palo_scan_range.__set_tablet_id(kMissingTabletId); + palo_scan_range.__set_version("114514"); + scan_range.scan_range.__set_palo_scan_range(palo_scan_range); + scan_ranges.push_back(scan_range); + + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({kMissingTabletId, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + cache_param.__set_allow_incremental(true); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + { + // A stale entry with incremental allowed, but the tablet does not exist + // in this test process: the decision falls back to a full recompute and + // reports why in the query profile. + CacheResult result; + result.push_back(std::make_unique()); + *result.back() = ColumnHelper::create_block({100, 200}); + query_cache->insert(cache_key, 100, result, {0}, 1); + } + + source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); + create_local_state(); + + EXPECT_FALSE(source_local_state->_is_incremental); + EXPECT_TRUE(source_local_state->_need_insert_cache); + const std::string* reason = + source_local_state->custom_profile()->get_info_string("IncrementalFallbackReason"); + ASSERT_NE(reason, nullptr); + EXPECT_EQ(*reason, "tablet not found"); +} + +TEST_F(QueryCacheOperatorTest, test_missing_runtime_fails_init) { + sink = std::make_unique(); + source = std::make_unique(); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + + // No runtime injected: a pass-through here could silently drop data if + // the paired scan still made a HIT decision, so init must fail loudly. + source->_cache_param = cache_param; + shared_state = sink->create_shared_state(); + source_local_state_uptr = CacheSourceLocalState::create_unique(state.get(), source.get()); + source_local_state = source_local_state_uptr.get(); + source_local_state->_global_cache = query_cache; + LocalStateInfo info {.parent_profile = &profile, + .scan_ranges = scan_ranges, + .shared_state = shared_state.get(), + .shared_state_map = {}, + .task_idx = 0}; + Status st = source_local_state_uptr->init(state.get(), info); + EXPECT_FALSE(st.ok()); + EXPECT_TRUE(st.to_string().find("query cache runtime is absent at the cache source") != + std::string::npos) + << st.to_string(); + EXPECT_EQ(query_cache->get_element_count(), 0); +} + +TEST_F(QueryCacheOperatorTest, test_hit_cache_multi_block_reordered_slots) { + // Two normalized-equivalent queries can share one digest with different + // output slot orders, so the entry is served through a column + // permutation. With MORE THAN ONE cached block the permutation must be + // applied to each cached block before merging it into the reused output + // block: the pipeline's clear_column_data() keeps the already-permuted + // schema, so merging a cached-order block into it positionally breaks -- + // a type error for heterogeneous slots like these (BIGINT/INT swapped), + // silently misplaced data for same-typed ones. + sink = std::make_unique(); + source = std::make_unique(); + EXPECT_TRUE(source->set_child(child_op)); + auto* row_desc = new MockRowDescriptor { + {std::make_shared(), std::make_shared()}, &pool}; + child_op->_mock_row_desc.reset(row_desc); + // The mock descriptor leaves every slot id at its default; give the two + // output slots distinct ids so the slot-order mapping is meaningful. + auto& slots = static_cast(row_desc->tuple_desc_map.front())->Slots; + slots[0]->_id = 100; + slots[1]->_id = 101; + + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[100] = 10; + cache_param.output_slot_mapping[101] = 11; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + + { + // The entry was written by the sibling query whose output order was + // (INT32 slot 11, INT64 slot 10) -- the reverse of this query. + int64_t version = 0; + std::string cache_key; + EXPECT_TRUE( + QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + CacheResult result; + result.push_back(std::make_unique()); + *result.back() = Block({ColumnHelper::create_column_with_name({1, 2}), + ColumnHelper::create_column_with_name({10, 20})}); + result.push_back(std::make_unique()); + *result.back() = Block({ColumnHelper::create_column_with_name({3}), + ColumnHelper::create_column_with_name({30})}); + query_cache->insert(cache_key, version, result, {11, 10}, 1); + } + + source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); + // In production this operator is also built without a plan node, so its + // _row_descriptor reports zero materialized slots and clear_column_data() + // wipes the block every pull -- the schema-carrying reused-block shape is + // accidentally unreachable today. Report a real slot count here to pin + // the permute-before-merge invariant against the natural cleanups (a real + // row descriptor, or removing the redundant per-operator wipe) that would + // make that shape live. + source->_row_descriptor._num_materialized_slots = 2; + create_local_state(); + + EXPECT_EQ(source_local_state->_slot_orders, (std::vector {10, 11})); + EXPECT_EQ(source_local_state->_hit_cache_column_orders, (std::vector {1, 0})); + + { + auto block = ColumnHelper::create_block({0}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + // ONE output block reused across pulls, exactly like the pipeline driver + // (PipelineTask feeds the same block every iteration and + // clear_column_data() keeps its schema): pull 2 is the shape that broke, + // because the reused block already carries the permuted schema. + Block block; + { + // First cached block, permuted to this query's order. + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_FALSE(eos); + Block expected({ColumnHelper::create_column_with_name({10, 20}), + ColumnHelper::create_column_with_name({1, 2})}); + EXPECT_TRUE(ColumnHelper::block_equal(block, expected)) << block.dump_data(); + } + { + // Second cached block through the schema-carrying reused block. + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_FALSE(eos); + Block expected({ColumnHelper::create_column_with_name({30}), + ColumnHelper::create_column_with_name({3})}); + EXPECT_TRUE(ColumnHelper::block_equal(block, expected)) << block.dump_data(); + } +} + +TEST_F(QueryCacheOperatorTest, test_incremental_reordered_write_back) { + // An INCREMENTAL merge through a permuted entry: the cached blocks are + // emitted in this query's slot order and the written-back entry must + // hold "cached + delta" under this query's slot order as one consistent + // whole, so a later exact hit through it permutes correctly again. + sink = std::make_unique(); + source = std::make_unique(); + EXPECT_TRUE(source->set_child(child_op)); + auto* row_desc = new MockRowDescriptor { + {std::make_shared(), std::make_shared()}, &pool}; + child_op->_mock_row_desc.reset(row_desc); + auto& slots = static_cast(row_desc->tuple_desc_map.front())->Slots; + slots[0]->_id = 100; + slots[1]->_id = 101; + + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[100] = 10; + cache_param.output_slot_mapping[101] = 11; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + cache_param.__set_allow_incremental(true); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + { + // Two stale cached blocks written by the reverse-ordered sibling. + CacheResult result; + result.push_back(std::make_unique()); + *result.back() = Block({ColumnHelper::create_column_with_name({1, 2}), + ColumnHelper::create_column_with_name({10, 20})}); + result.push_back(std::make_unique()); + *result.back() = Block({ColumnHelper::create_column_with_name({3}), + ColumnHelper::create_column_with_name({30})}); + query_cache->insert(cache_key, 100, result, {11, 10}, 1, /*delta_count=*/0); + } + + source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); + { + auto decision = std::make_shared(); + decision->mode = QueryCacheInstanceDecision::Mode::INCREMENTAL; + decision->key_valid = true; + decision->cache_key = cache_key; + decision->current_version = version; + decision->cached_version = 100; + decision->cached_delta_count = 0; + EXPECT_TRUE(query_cache->lookup_any_version(cache_key, &decision->handle)); + source->_query_cache_runtime->inject_decision_for_test(cache_key, decision); + } + // Pin the schema-carrying reused-block shape; see the rationale in + // test_hit_cache_multi_block_reordered_slots. + source->_row_descriptor._num_materialized_slots = 2; + create_local_state(); + + EXPECT_TRUE(source_local_state->_is_incremental); + EXPECT_TRUE(source_local_state->_need_insert_cache); + EXPECT_EQ(source_local_state->_hit_cache_column_orders, (std::vector {1, 0})); + + { + // The delta partial result arrives in THIS query's slot order. + auto block = Block({ColumnHelper::create_column_with_name({40}), + ColumnHelper::create_column_with_name({4})}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + // ONE output block reused across pulls, like the pipeline driver: the + // second cached pull must permute correctly into the schema-carrying + // reused block before its write-back snapshot is taken. + Block block; + for (int i = 0; i < 2; ++i) { + // Both cached blocks flow out permuted, without merge errors. + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_FALSE(eos); + } + // The write-back snapshots are zero-copy views over the pinned entry's + // columns (in this query's order), not payload copies: the single + // materialization happens at insert time. + const auto& pinned_blocks = *source_local_state->_cache_decision->handle.get_cache_result(); + const auto& kept_blocks = source_local_state->_local_cache_blocks; + ASSERT_EQ(kept_blocks.size(), 2); + const auto* pinned_first_int64 = pinned_blocks[0]->get_by_position(1).column.get(); + EXPECT_EQ(kept_blocks[0]->get_by_position(0).column.get(), pinned_first_int64); + EXPECT_EQ(kept_blocks[0]->get_by_position(1).column.get(), + pinned_blocks[0]->get_by_position(0).column.get()); + EXPECT_EQ(kept_blocks[1]->get_by_position(0).column.get(), + pinned_blocks[1]->get_by_position(1).column.get()); + { + // Then the delta. + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 1); + } + + // The merged entry holds every block under THIS query's slot order. + doris::QueryCacheHandle handle; + EXPECT_TRUE(query_cache->lookup(cache_key, 114514, &handle)); + EXPECT_EQ(handle.get_cache_delta_count(), 1); + EXPECT_EQ(*handle.get_cache_slot_orders(), (std::vector {10, 11})); + const auto& cached_blocks = *handle.get_cache_result(); + EXPECT_EQ(cached_blocks.size(), 3); + // The insert materialized one cache-owned copy: the new entry does not + // alias the old pinned columns. + EXPECT_NE(cached_blocks[0]->get_by_position(0).column.get(), pinned_first_int64); + Block expected0({ColumnHelper::create_column_with_name({10, 20}), + ColumnHelper::create_column_with_name({1, 2})}); + EXPECT_TRUE(ColumnHelper::block_equal(*cached_blocks[0], expected0)) + << cached_blocks[0]->dump_data(); + Block expected1({ColumnHelper::create_column_with_name({30}), + ColumnHelper::create_column_with_name({3})}); + EXPECT_TRUE(ColumnHelper::block_equal(*cached_blocks[1], expected1)) + << cached_blocks[1]->dump_data(); + Block expected2({ColumnHelper::create_column_with_name({40}), + ColumnHelper::create_column_with_name({4})}); + EXPECT_TRUE(ColumnHelper::block_equal(*cached_blocks[2], expected2)) + << cached_blocks[2]->dump_data(); + + { + // A spurious re-poll after eos must not re-publish: the commit is + // latched off after the insert, so with the write-back set already + // cleared the merged entry keeps its three blocks instead of being + // replaced by an EMPTY set under the same version. + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + doris::QueryCacheHandle repoll_handle; + EXPECT_TRUE(query_cache->lookup(cache_key, 114514, &repoll_handle)); + EXPECT_EQ(repoll_handle.get_cache_result()->size(), 3); + } +} + +TEST_F(QueryCacheOperatorTest, test_failed_final_delta_merge_publishes_nothing) { + // On the final delta block, eos is observed BEFORE the block is merged: + // if that merge fails, the entry must NOT be committed, or an incomplete + // prefix (cached blocks plus earlier deltas, missing the failed block) + // would be served as the current version by later exact hits. The + // malformed two-column final block makes the merge fail determinately. + sink = std::make_unique(); + source = std::make_unique(); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + cache_param.__set_allow_incremental(true); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + { + CacheResult result; + result.push_back(std::make_unique()); + *result.back() = ColumnHelper::create_block({1, 2, 3}); + query_cache->insert(cache_key, 100, result, {0}, 1, /*delta_count=*/0); + } + + source->_cache_param = cache_param; + source->_query_cache_runtime = std::make_shared(cache_param, query_cache); + { + auto decision = std::make_shared(); + decision->mode = QueryCacheInstanceDecision::Mode::INCREMENTAL; + decision->key_valid = true; + decision->cache_key = cache_key; + decision->current_version = version; + decision->cached_version = 100; + decision->cached_delta_count = 0; + EXPECT_TRUE(query_cache->lookup_any_version(cache_key, &decision->handle)); + source->_query_cache_runtime->inject_decision_for_test(cache_key, decision); + } + // Pin the schema-carrying reused-block shape (rationale in + // test_hit_cache_multi_block_reordered_slots): with the default empty row + // descriptor the reused block is wiped every pull and re-cloned from the + // incoming block, so the malformed final block would merge into a clone + // of itself and SUCCEED, unbinding this test from the bug it guards. + source->_row_descriptor._num_materialized_slots = 1; + create_local_state(); + EXPECT_TRUE(source_local_state->_need_insert_cache); + + { + // A good delta block, then a malformed final one. + auto good = ColumnHelper::create_block({6, 7}); + EXPECT_TRUE(sink->sink(state.get(), &good, false).ok()); + auto bad = Block({ColumnHelper::create_column_with_name({8}), + ColumnHelper::create_column_with_name({9})}); + EXPECT_TRUE(sink->sink(state.get(), &bad, true).ok()); + } + Block block; + { + // Cached block, then the good delta block. + bool eos = false; + EXPECT_TRUE(source->get_block(state.get(), &block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_TRUE(source->get_block(state.get(), &block, &eos).ok()); + EXPECT_FALSE(eos); + } + { + // The final block: eos flips to true first, then the merge fails. + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_FALSE(st.ok()); + } + + // Nothing was committed: no entry at the current version, and the stale + // entry still carries version 100 untouched. + doris::QueryCacheHandle handle; + EXPECT_FALSE(query_cache->lookup(cache_key, 114514, &handle)); + doris::QueryCacheHandle stale_handle; + EXPECT_TRUE(query_cache->lookup_any_version(cache_key, &stale_handle)); + EXPECT_EQ(stale_handle.get_cache_version(), 100); +} + } // namespace doris diff --git a/be/test/exec/operator/union_operator_test.cpp b/be/test/exec/operator/union_operator_test.cpp index 3009e977a47d8f..8590c86f732789 100644 --- a/be/test/exec/operator/union_operator_test.cpp +++ b/be/test/exec/operator/union_operator_test.cpp @@ -254,7 +254,9 @@ TEST_F(UnionOperatorTest, test_sink_and_source) { { for (int i = 0; i < child_size; i++) { - sink_state[i]->_batch_size = 2; + // Each sink input should be queued immediately, including materialized children whose + // input is smaller than the runtime batch size. + sink_state[i]->_batch_size = 10; Block block = ColumnHelper::create_block({1, 2}, {3, 4}); EXPECT_TRUE(sink_ops[i]->sink(sink_state[i].get(), &block, false)); } @@ -291,4 +293,4 @@ TEST_F(UnionOperatorTest, test_sink_and_source) { block, ColumnHelper::create_block({1, 2}, {3, 4}))); } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/exec/pipeline/data_queue_test.cpp b/be/test/exec/pipeline/data_queue_test.cpp index f8ba26bc664d01..db1c3cccef4f0a 100644 --- a/be/test/exec/pipeline/data_queue_test.cpp +++ b/be/test/exec/pipeline/data_queue_test.cpp @@ -221,57 +221,59 @@ class DataQueueTest : public testing::Test { // Initial state: no data, no finish. TEST_F(DataQueueTest, InitialState) { - EXPECT_FALSE(data_queue->has_more_data()); - EXPECT_FALSE(data_queue->is_all_finish()); - EXPECT_FALSE(data_queue->remaining_has_data()); + EXPECT_EQ(data_queue->debug_string(), "(is_all_finish = false, has_data = false)"); + auto queue_block = TEST_TRY(data_queue->get_block_from_queue()); + EXPECT_EQ(queue_block.block, nullptr); + EXPECT_FALSE(queue_block.eos); } // Push one block and retrieve it. TEST_F(DataQueueTest, SinglePushPop) { - EXPECT_TRUE(data_queue->push_block(make_block(), 0).ok()); - EXPECT_TRUE(data_queue->has_more_data()); + EXPECT_TRUE(data_queue->push_block(make_block(), 0, false).ok()); + EXPECT_EQ(data_queue->debug_string(), "(is_all_finish = false, has_data = true)"); - // Find the queue with data. - EXPECT_TRUE(data_queue->remaining_has_data()); - - std::unique_ptr out; - int child_idx = -1; - EXPECT_TRUE(data_queue->get_block_from_queue(&out, &child_idx).ok()); - EXPECT_NE(out, nullptr); - EXPECT_EQ(child_idx, 0); - EXPECT_FALSE(data_queue->has_more_data()); + auto queue_block = TEST_TRY(data_queue->get_block_from_queue()); + EXPECT_NE(queue_block.block, nullptr); + EXPECT_FALSE(queue_block.eos); + EXPECT_EQ(data_queue->debug_string(), "(is_all_finish = false, has_data = false)"); } -// is_all_finish only becomes true after all children call set_finish. +// is_all_finish only becomes true after all children push eos. TEST_F(DataQueueTest, IsAllFinishAfterAllChildren) { - data_queue->set_finish(0); - EXPECT_FALSE(data_queue->is_all_finish()); - data_queue->set_finish(1); - EXPECT_FALSE(data_queue->is_all_finish()); - data_queue->set_finish(2); - EXPECT_TRUE(data_queue->is_all_finish()); + EXPECT_TRUE(data_queue->push_block(nullptr, 0, true).ok()); + auto first_queue_block = TEST_TRY(data_queue->get_block_from_queue()); + EXPECT_FALSE(first_queue_block.eos); + EXPECT_TRUE(data_queue->push_block(nullptr, 1, true).ok()); + auto second_queue_block = TEST_TRY(data_queue->get_block_from_queue()); + EXPECT_FALSE(second_queue_block.eos); + EXPECT_TRUE(data_queue->push_block(nullptr, 2, true).ok()); + auto last_queue_block = TEST_TRY(data_queue->get_block_from_queue()); + EXPECT_TRUE(last_queue_block.eos); } -// set_finish is idempotent. -TEST_F(DataQueueTest, SetFinishIdempotent) { - data_queue->set_finish(0); - data_queue->set_finish(0); // second call must not double-decrement - data_queue->set_finish(1); - data_queue->set_finish(2); - EXPECT_TRUE(data_queue->is_all_finish()); +// eos push is idempotent. +TEST_F(DataQueueTest, EosPushIdempotent) { + EXPECT_TRUE(data_queue->push_block(nullptr, 0, true).ok()); + EXPECT_TRUE(data_queue->push_block(nullptr, 0, true).ok()); + EXPECT_TRUE(data_queue->push_block(nullptr, 1, true).ok()); + EXPECT_TRUE(data_queue->push_block(nullptr, 2, true).ok()); + auto queue_block = TEST_TRY(data_queue->get_block_from_queue()); + EXPECT_TRUE(queue_block.eos); } -// child_idx returned by get_block_from_queue reflects the actual queue. -TEST_F(DataQueueTest, ChildIdxReturned) { +// DataQueueBlock can return a popped block to the originating child free list. +TEST_F(DataQueueTest, ReturnBlockToChildFreeList) { // Push to child 1 only. - EXPECT_TRUE(data_queue->push_block(make_block(), 1).ok()); - data_queue->remaining_has_data(); // advance _flag_queue_idx to find child 1 + EXPECT_TRUE(data_queue->push_block(make_block(), 1, false).ok()); - std::unique_ptr out; - int child_idx = -1; - EXPECT_TRUE(data_queue->get_block_from_queue(&out, &child_idx).ok()); - EXPECT_NE(out, nullptr); - EXPECT_EQ(child_idx, 1); + auto queue_block = TEST_TRY(data_queue->get_block_from_queue()); + EXPECT_NE(queue_block.block, nullptr); + queue_block.block->clear(); + auto* returned_block = queue_block.block.get(); + data_queue->push_free_block(std::move(queue_block)); + + auto reused_block = data_queue->get_free_block(1); + EXPECT_EQ(reused_block.get(), returned_block); } // get_free_block returns a new block when free list is empty, reuses when not. @@ -282,7 +284,9 @@ TEST_F(DataQueueTest, FreeBlockReuse) { // Return it to the free list. block->clear(); // ensure rows == 0 - data_queue->push_free_block(std::move(block), 0); + DataQueueBlock queue_block; + queue_block.block = std::move(block); + data_queue->push_free_block(std::move(queue_block)); // Second call: must return the recycled block. auto block2 = data_queue->get_free_block(0); @@ -292,7 +296,9 @@ TEST_F(DataQueueTest, FreeBlockReuse) { // In low-memory mode push_free_block discards blocks and max drops to 1. TEST_F(DataQueueTest, LowMemoryMode) { // Pre-populate the free list. - data_queue->push_free_block(Block::create_unique(), 0); + DataQueueBlock queue_block; + queue_block.block = Block::create_unique(); + data_queue->push_free_block(std::move(queue_block)); data_queue->set_low_memory_mode(); @@ -303,7 +309,9 @@ TEST_F(DataQueueTest, LowMemoryMode) { // push_free_block now discards. block->clear(); - data_queue->push_free_block(std::move(block), 0); + DataQueueBlock low_memory_block; + low_memory_block.block = std::move(block); + data_queue->push_free_block(std::move(low_memory_block)); auto block2 = data_queue->get_free_block(0); // Still gets a fresh allocation (free list stays empty). EXPECT_NE(block2, nullptr); @@ -311,15 +319,17 @@ TEST_F(DataQueueTest, LowMemoryMode) { // terminate() finishes all children and clears pending blocks from sub-queues. TEST_F(DataQueueTest, Terminate) { - EXPECT_TRUE(data_queue->push_block(make_block(), 0).ok()); - EXPECT_TRUE(data_queue->push_block(make_block(), 1).ok()); + EXPECT_TRUE(data_queue->push_block(make_block(), 0, false).ok()); + EXPECT_TRUE(data_queue->push_block(make_block(), 1, false).ok()); data_queue->terminate(); - EXPECT_TRUE(data_queue->is_all_finish()); - // remaining_has_data() checks blocks_in_queue per sub-queue, - // which clear_blocks() resets to 0. - EXPECT_FALSE(data_queue->remaining_has_data()); + EXPECT_EQ(data_queue->debug_string(), "(is_all_finish = true, has_data = false)"); + source_dep->block(); + EXPECT_TRUE(source_dep->ready()); + auto queue_block = TEST_TRY(data_queue->get_block_from_queue()); + EXPECT_EQ(queue_block.block, nullptr); + EXPECT_TRUE(queue_block.eos); } // set_max_blocks_in_sub_queue propagates to every sub-queue. @@ -327,12 +337,12 @@ TEST_F(DataQueueTest, SetMaxBlocksInSubQueue) { data_queue->set_max_blocks_in_sub_queue(5); // Push 5 blocks to child 0 — sink must stay ready (not over the limit yet). for (int i = 0; i < 5; i++) { - EXPECT_TRUE(data_queue->push_block(make_block(), 0).ok()); + EXPECT_TRUE(data_queue->push_block(make_block(), 0, false).ok()); } EXPECT_TRUE(sink_deps[0]->ready()); // 6th push exceeds limit → sink blocked. - EXPECT_TRUE(data_queue->push_block(make_block(), 0).ok()); + EXPECT_TRUE(data_queue->push_block(make_block(), 0, false).ok()); EXPECT_FALSE(sink_deps[0]->ready()); } @@ -341,10 +351,28 @@ TEST_F(DataQueueTest, SourceReadyOnPush) { source_dep->block(); // start blocked EXPECT_FALSE(source_dep->ready()); - EXPECT_TRUE(data_queue->push_block(make_block(), 0).ok()); + EXPECT_TRUE(data_queue->push_block(make_block(), 0, false).ok()); EXPECT_TRUE(source_dep->ready()); } +TEST_F(DataQueueTest, SourceReadyOnEosOnly) { + source_dep->block(); + EXPECT_FALSE(source_dep->ready()); + + EXPECT_TRUE(data_queue->push_block(nullptr, 0, true).ok()); + EXPECT_TRUE(source_dep->ready()); +} + +TEST_F(DataQueueTest, EosOnlyAfterAllChildren) { + EXPECT_TRUE(data_queue->push_block(nullptr, 0, true).ok()); + EXPECT_TRUE(data_queue->push_block(nullptr, 1, true).ok()); + EXPECT_TRUE(data_queue->push_block(nullptr, 2, true).ok()); + + auto queue_block = TEST_TRY(data_queue->get_block_from_queue()); + EXPECT_EQ(queue_block.block, nullptr); + EXPECT_TRUE(queue_block.eos); +} + // --------------------------------------------------------------------------- // Multi-threaded integration test (existing) // --------------------------------------------------------------------------- @@ -355,19 +383,9 @@ TEST_F(DataQueueTest, MultiTest) { while (true) { bool eos = false; if (source_dep->ready()) { - Defer set_eos {[&]() { - if (data_queue->remaining_has_data()) { - eos = false; - } else if (data_queue->is_all_finish()) { - eos = !data_queue->remaining_has_data(); - } else { - eos = false; - } - }}; - std::unique_ptr output_block; - int child_idx = 0; - EXPECT_TRUE(data_queue->get_block_from_queue(&output_block, &child_idx)); - if (output_block) { + auto queue_block = TEST_TRY(data_queue->get_block_from_queue()); + eos = queue_block.eos; + if (queue_block.block) { output_count++; } } @@ -392,11 +410,11 @@ TEST_F(DataQueueTest, MultiTest) { int i = 0; while (i < 50) { if (sink_deps[id]->ready()) { - EXPECT_TRUE(data_queue->push_block(std::move(input_blocks[id][i]), id).ok()); + EXPECT_TRUE(data_queue->push_block(std::move(input_blocks[id][i]), id, false).ok()); i++; } } - data_queue->set_finish(id); + EXPECT_TRUE(data_queue->push_block(nullptr, id, true).ok()); }; std::thread input1(input_func, 0); @@ -409,7 +427,7 @@ TEST_F(DataQueueTest, MultiTest) { output1.join(); EXPECT_EQ(output_count, 150); - EXPECT_TRUE(data_queue->is_all_finish()); + EXPECT_EQ(data_queue->debug_string(), "(is_all_finish = true, has_data = false)"); } // ./run-be-ut.sh --run --filter=DataQueueTest.* diff --git a/be/test/exec/pipeline/query_cache_fragment_context_test.cpp b/be/test/exec/pipeline/query_cache_fragment_context_test.cpp new file mode 100644 index 00000000000000..f00fcc9735d263 --- /dev/null +++ b/be/test/exec/pipeline/query_cache_fragment_context_test.cpp @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Reaches the private _create_operator directly: constructing a full prepare() +// payload for one switch branch would couple the test to every unrelated +// prepare step. BE UT compiles with -fno-access-control (see be/test/AGENTS.md), +// so no access hack is needed. + +#include +#include +#include +#include +#include + +#include + +#include "exec/pipeline/pipeline.h" +#include "exec/pipeline/pipeline_fragment_context.h" +#include "runtime/exec_env.h" +#include "runtime/query_context.h" + +namespace doris { + +static void empty_finish_function(RuntimeState*, Status*) {} + +// The plan tree is built in pre-order, so when the query cache is enabled the +// cache source (above the scan) must have created the shared QueryCacheRuntime +// before the scan node is reached. A missing runtime means FE sent a malformed +// plan shape; silently creating one here would let a HIT decision skip the +// scan while no cache source emits the entry, i.e. drop data, so the operator +// factory must fail loudly instead. +class QueryCacheFragmentContextTest : public testing::Test { +protected: + void SetUp() override { + TQueryOptions query_options; + TNetworkAddress fe_address; + fe_address.hostname = "127.0.0.1"; + fe_address.port = 8060; + _query_ctx = + QueryContext::create(_query_id, ExecEnv::GetInstance(), query_options, fe_address, + true, fe_address, QuerySource::INTERNAL_FRONTEND); + + TQueryCacheParam cache_param; + cache_param.__set_node_id(0); + cache_param.__set_digest("test_digest"); + _params.fragment.__set_query_cache_param(cache_param); + _context = std::make_shared( + _query_id, _params, _query_ctx, ExecEnv::GetInstance(), empty_finish_function); + } + + Status create_scan_operator(const TPlanNode& tnode) { + ObjectPool pool; + DescriptorTbl descs; + OperatorPtr op; + OperatorPtr cache_op; + PipelinePtr pipe = std::make_shared(0, 1, 1); + return _context->_create_operator(&pool, tnode, descs, op, pipe, /*parent_idx=*/-1, + /*child_idx=*/0, + /*followed_by_shuffled_operator=*/false, + /*require_bucket_distribution=*/false, cache_op); + } + + TUniqueId _query_id; + TPipelineFragmentParams _params; + std::shared_ptr _query_ctx; + std::shared_ptr _context; +}; + +TEST_F(QueryCacheFragmentContextTest, scan_without_runtime_fails_loudly) { + TPlanNode tnode; + tnode.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + tnode.__set_node_id(7); + tnode.__set_num_children(0); + + ASSERT_EQ(_context->_query_cache_runtime, nullptr); + Status st = create_scan_operator(tnode); + EXPECT_FALSE(st.ok()); + // The full message pins the scan-side wording and the argument order + // (tnode.node_id first, then the FE-designated cache node id). + EXPECT_TRUE(st.to_string().find("query cache runtime is absent at the scan node, " + "node_id=7, cache node_id=0") != std::string::npos) + << st.to_string(); +} + +TEST_F(QueryCacheFragmentContextTest, binlog_scan_without_runtime_fails_before_binlog_handling) { + // The runtime check comes first: a row-binlog scan with a broken setup + // must fail the same way instead of dereferencing the missing runtime in + // disable_for_binlog_scan(). + TPlanNode tnode; + tnode.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + tnode.__set_node_id(7); + tnode.__set_num_children(0); + TOlapScanNode olap_scan_node; + olap_scan_node.__set_read_row_binlog(true); + tnode.__set_olap_scan_node(olap_scan_node); + + ASSERT_EQ(_context->_query_cache_runtime, nullptr); + Status st = create_scan_operator(tnode); + EXPECT_FALSE(st.ok()); + EXPECT_TRUE(st.to_string().find("query cache runtime is absent at the scan node, " + "node_id=7, cache node_id=0") != std::string::npos) + << st.to_string(); +} + +} // namespace doris diff --git a/be/test/exec/pipeline/query_cache_test.cpp b/be/test/exec/pipeline/query_cache_test.cpp index ec0f8c6c697686..4f79f4e8943146 100644 --- a/be/test/exec/pipeline/query_cache_test.cpp +++ b/be/test/exec/pipeline/query_cache_test.cpp @@ -17,13 +17,39 @@ #include "runtime/query_cache/query_cache.h" +#include +#include +#include #include +#include +#include +#include +#include #include +#include +#include #include +#include "cloud/config.h" +#include "common/config.h" +#include "common/metrics/doris_metrics.h" #include "core/data_type/data_type_number.h" +#include "cpp/sync_point.h" +#include "io/fs/local_file_system.h" +#include "json2pb/json_to_pb.h" +#include "storage/data_dir.h" +#include "storage/options.h" +#include "storage/rowset/rowset_meta.h" +#include "storage/storage_engine.h" +#include "storage/tablet/base_tablet.h" +#include "storage/tablet/tablet.h" +#include "storage/tablet/tablet_manager.h" +#include "storage/tablet/tablet_meta.h" #include "testutil/column_helper.h" +#include "util/debug_points.h" +#include "util/defer_op.h" +#include "util/uid_util.h" namespace doris { class QueryCacheTest : public testing::Test { @@ -287,4 +313,831 @@ TEST_F(QueryCacheTest, insert_and_lookup) { // ./run-be-ut.sh --run --filter=DataQueueTest.* +namespace { + +std::vector make_scan_ranges(int64_t tablet_id, const std::string& version) { + std::vector scan_ranges; + TScanRangeParams scan_range; + TPaloScanRange palo_scan_range; + palo_scan_range.__set_tablet_id(tablet_id); + palo_scan_range.__set_version(version); + scan_range.scan_range.__set_palo_scan_range(palo_scan_range); + scan_ranges.push_back(scan_range); + return scan_ranges; +} + +TQueryCacheParam make_cache_param(int64_t tablet_id) { + TQueryCacheParam cache_param; + cache_param.__set_digest("runtime_test_digest"); + cache_param.tablet_to_range.insert({tablet_id, "range"}); + // FE always sets the entry limits; keep them roomy so decision tests do + // not trip the write-back feasibility check unintentionally. + cache_param.__set_entry_max_bytes(1024 * 1024); + cache_param.__set_entry_max_rows(100000); + return cache_param; +} + +void insert_entry(QueryCache* cache, const std::string& cache_key, int64_t version, + int64_t delta_count) { + CacheResult result; + result.push_back(std::make_unique()); + *result.back() = ColumnHelper::create_block({1, 2, 3}); + cache->insert(cache_key, version, result, {0}, 1, delta_count); +} + +} // namespace + +TEST_F(QueryCacheTest, runtime_decision_miss_and_idempotent) { + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); + auto scan_ranges = make_scan_ranges(42, "100"); + QueryCacheRuntime runtime(make_cache_param(42), cache.get()); + + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_EQ(decision->current_version, 100); + EXPECT_FALSE(decision->handle.valid()); + + // Idempotent: the second caller (e.g. the other operator) observes the + // same decision object. + auto decision2 = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision.get(), decision2.get()); +} + +TEST_F(QueryCacheTest, runtime_decision_invalid_key) { + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); + auto scan_ranges = make_scan_ranges(42, "100"); + // Tablet 42 is missing from tablet_to_range, so build_cache_key fails and + // the query degrades to an uncached scan instead of failing. + TQueryCacheParam cache_param; + cache_param.__set_digest("runtime_test_digest"); + cache_param.tablet_to_range.insert({43, "range"}); + QueryCacheRuntime runtime(cache_param, cache.get()); + + int64_t stale_before = DorisMetrics::instance()->query_cache_stale_hit_total->value(); + int64_t fallback_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_FALSE(decision->key_valid); + // Every caller shares one immutable invalid decision (and one log line). + EXPECT_EQ(decision.get(), runtime.get_or_make_decision(scan_ranges).get()); + // The degraded-scan path returns before the candidate build, so it must + // settle no metrics at all. + EXPECT_EQ(DorisMetrics::instance()->query_cache_stale_hit_total->value(), stale_before); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallback_before); +} + +TEST_F(QueryCacheTest, runtime_decision_hit) { + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); + auto scan_ranges = make_scan_ranges(42, "100"); + auto cache_param = make_cache_param(42); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(cache.get(), cache_key, 100, 0); + + int64_t stale_before = DorisMetrics::instance()->query_cache_stale_hit_total->value(); + int64_t fallback_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + QueryCacheRuntime runtime(cache_param, cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::HIT); + EXPECT_TRUE(decision->handle.valid()); + EXPECT_EQ(decision->handle.get_cache_version(), 100); + // An exact hit is neither a stale hit nor a fallback. + EXPECT_EQ(DorisMetrics::instance()->query_cache_stale_hit_total->value(), stale_before); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallback_before); +} + +TEST_F(QueryCacheTest, runtime_decision_force_refresh) { + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); + auto scan_ranges = make_scan_ranges(42, "100"); + auto cache_param = make_cache_param(42); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(cache.get(), cache_key, 100, 0); + + cache_param.__set_force_refresh_query_cache(true); + int64_t fallback_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + QueryCacheRuntime runtime(cache_param, cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + // Recompute and write back even though a fresh entry exists. + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_FALSE(decision->handle.valid()); + // A forced refresh is a deliberate MISS with an empty reason, not a + // fallback: the caller-side settlement must not count it. + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallback_before); +} + +TEST_F(QueryCacheTest, runtime_decision_binlog_scan) { + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); + auto scan_ranges = make_scan_ranges(42, "100"); + auto cache_param = make_cache_param(42); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(cache.get(), cache_key, 100, 0); + + int64_t fallback_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + QueryCacheRuntime runtime(cache_param, cache.get()); + runtime.disable_for_binlog_scan(); + auto decision = runtime.get_or_make_decision(scan_ranges); + // A binlog scan must neither serve the cached entry nor write back, and + // its deliberate MISS (empty reason) must not count as a fallback. + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_FALSE(decision->key_valid); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallback_before); +} + +TEST_F(QueryCacheTest, runtime_decision_stale_without_incremental) { + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); + auto scan_ranges = make_scan_ranges(42, "100"); + auto cache_param = make_cache_param(42); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(cache.get(), cache_key, 50, 0); + + // allow_incremental unset: a stale entry is a plain miss (full recompute). + int64_t fallback_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + QueryCacheRuntime runtime(cache_param, cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_FALSE(decision->handle.valid()); + // Incremental merge never engaged (empty reason), so the caller-side + // settlement must not count a fallback. + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallback_before); +} + +TEST_F(QueryCacheTest, runtime_decision_stale_incremental_fallbacks) { + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); + auto scan_ranges = make_scan_ranges(42, "100"); + auto cache_param = make_cache_param(42); + cache_param.__set_allow_incremental(true); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + + { + // The entry is newer than the version this replica is asked to read: + // never usable, fall back to a full scan of the requested version. + insert_entry(cache.get(), cache_key, 200, 0); + QueryCacheRuntime runtime(cache_param, cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_EQ(decision->incremental_fallback_reason, "cached entry is newer"); + } + { + // Too many accumulated incremental merges: force a full recompute to + // compact the entry. (Checked before any tablet access.) + insert_entry(cache.get(), cache_key, 50, config::query_cache_max_incremental_merge_count); + QueryCacheRuntime runtime(cache_param, cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_EQ(decision->incremental_fallback_reason, + "delta count reached compaction threshold"); + } + { + // Use a tablet id that exists nowhere, so fetching the tablet fails + // regardless of whether some other suite left a storage engine behind + // in this test process, and the decision safely falls back to MISS. + auto missing_scan_ranges = make_scan_ranges(424242424242, "100"); + auto missing_cache_param = make_cache_param(424242424242); + missing_cache_param.__set_allow_incremental(true); + std::string missing_cache_key; + int64_t missing_version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(missing_scan_ranges, missing_cache_param, + &missing_cache_key, &missing_version) + .ok()); + insert_entry(cache.get(), missing_cache_key, 50, 0); + QueryCacheRuntime runtime(missing_cache_param, cache.get()); + auto decision = runtime.get_or_make_decision(missing_scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_FALSE(decision->handle.valid()); + EXPECT_EQ(decision->incremental_fallback_reason, "tablet not found"); + } +} + +TEST_F(QueryCacheTest, lookup_any_version_and_delta_count) { + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); + auto scan_ranges = make_scan_ranges(42, "100"); + auto cache_param = make_cache_param(42); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + + QueryCacheHandle handle; + EXPECT_FALSE(cache->lookup_any_version(cache_key, &handle)); + + int64_t write_backs_before = DorisMetrics::instance()->query_cache_write_back_total->value(); + insert_entry(cache.get(), cache_key, 50, 3); + EXPECT_EQ(DorisMetrics::instance()->query_cache_write_back_total->value(), + write_backs_before + 1); + QueryCacheHandle handle2; + EXPECT_TRUE(cache->lookup_any_version(cache_key, &handle2)); + EXPECT_EQ(handle2.get_cache_version(), 50); + EXPECT_EQ(handle2.get_cache_delta_count(), 3); + // insert_entry stores one 3-row block with cache_size 1. + EXPECT_EQ(handle2.get_cache_total_rows(), 3); + EXPECT_EQ(handle2.get_cache_total_bytes(), 1); + + // Exact-version lookup still rejects the stale entry. + QueryCacheHandle handle3; + EXPECT_FALSE(cache->lookup(cache_key, 100, &handle3)); +} + +TEST_F(QueryCacheTest, runtime_default_global_cache) { + // The single-argument constructor falls back to the global instance + // (whatever it is in this test environment). + QueryCacheRuntime runtime(make_cache_param(42)); + EXPECT_EQ(runtime.cache(), QueryCache::instance()); +} + +TEST_F(QueryCacheTest, take_delta_read_source) { + auto decision = std::make_shared(); + decision->_delta_read_sources[42] = std::make_unique(); + + EXPECT_EQ(decision->take_delta_read_source(41), nullptr); + auto source = decision->take_delta_read_source(42); + EXPECT_NE(source, nullptr); + // Each read source can be consumed exactly once. + EXPECT_EQ(decision->take_delta_read_source(42), nullptr); +} + +TEST_F(QueryCacheTest, runtime_decision_stale_incremental_cloud_mode) { + std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); + auto scan_ranges = make_scan_ranges(42, "100"); + auto cache_param = make_cache_param(42); + cache_param.__set_allow_incremental(true); + + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(cache.get(), cache_key, 50, 0); + + // Incremental merge only supports local storage for now. + std::string saved_deploy_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + QueryCacheRuntime runtime(cache_param, cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + config::deploy_mode = saved_deploy_mode; + + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud mode"); +} + +// Exercises the per-tablet part of the incremental decision against a real +// (metadata-only) tablet registered in a real storage engine: capturing the +// delta read source never touches segment files, so no data is needed. +class QueryCacheIncrementalTest : public testing::Test { +protected: + static constexpr int64_t kTabletId = 15673; + static constexpr const char* kTestDir = "/ut_dir/query_cache_incremental_test"; + + // Fatal assertions: every test in this fixture dereferences the storage + // engine and the data dir, so a failed setup must not fall through to the + // test body (TearDown still runs and tolerates the partial state). + void SetUp() override { + char buffer[1024]; + ASSERT_NE(getcwd(buffer, sizeof(buffer)), nullptr); + _absolute_dir = std::string(buffer) + kTestDir; + Status st = io::global_local_filesystem()->delete_directory(_absolute_dir); + ASSERT_TRUE(st.ok()) << st; + st = io::global_local_filesystem()->create_directory(_absolute_dir); + ASSERT_TRUE(st.ok()) << st; + + auto engine = std::make_unique(EngineOptions {}); + _engine = engine.get(); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + _data_dir = std::make_unique(*_engine, _absolute_dir); + st = _data_dir->init(); + ASSERT_TRUE(st.ok()) << st; + _cache.reset(QueryCache::create_global_cache(1024 * 1024)); + } + + void TearDown() override { + // Idempotent guard: a fatal assertion between enable_processing() and + // the in-test cleanup (see the racer tests) must not leak an enabled + // SyncPoint with callbacks over dead stack variables to later tests. + auto* sp = SyncPoint::get_instance(); + sp->disable_processing(); + sp->clear_all_call_backs(); + _cache.reset(); + _data_dir.reset(); + ExecEnv::GetInstance()->set_storage_engine(nullptr); + _engine = nullptr; + Status st = io::global_local_filesystem()->delete_directory(_absolute_dir); + EXPECT_TRUE(st.ok()) << st; + } + + void init_rs_meta(RowsetMetaSharedPtr& rs_meta, const TabletMetaSharedPtr& tablet_meta, + int64_t start, int64_t end, bool with_delete_predicate) { + static const std::string json_rowset_meta = R"({ + "rowset_id": 540081, + "tablet_id": 15673, + "txn_id": 4042, + "tablet_schema_hash": 567997577, + "rowset_type": "BETA_ROWSET", + "rowset_state": "VISIBLE", + "start_version": 2, + "end_version": 2, + "num_rows": 3929, + "total_disk_size": 84699, + "data_disk_size": 84464, + "index_disk_size": 235, + "empty": false, + "load_id": { + "hi": -5350970832824939812, + "lo": -6717994719194512122 + }, + "creation_time": 1553765670 + })"; + RowsetMetaPB rowset_meta_pb; + ASSERT_TRUE(json2pb::JsonToProtoMessage(json_rowset_meta, &rowset_meta_pb)); + rowset_meta_pb.set_start_version(start); + rowset_meta_pb.set_end_version(end); + // One distinct rowset id per rowset (the json template repeats one id): + // the merge-on-write history-rewrite check tells baseline and delta + // rowsets apart by rowset id, see rowset_id_for(). + rowset_meta_pb.set_rowset_id(540081 + start); + rowset_meta_pb.set_creation_time(10000); + if (with_delete_predicate) { + DeletePredicatePB* delete_predicate = rowset_meta_pb.mutable_delete_predicate(); + delete_predicate->set_version(static_cast(start)); + delete_predicate->add_sub_predicates("k1='1'"); + } + ASSERT_TRUE(rs_meta->init_from_pb(rowset_meta_pb)); + rs_meta->set_tablet_schema(tablet_meta->tablet_schema()); + } + + TabletSharedPtr create_tablet(TKeysType::type keys_type, bool enable_merge_on_write, + const std::vector>& versions, + int64_t delete_predicate_start_version = -1) { + TTabletSchema schema; + schema.keys_type = keys_type; + TabletMetaSharedPtr tablet_meta(new TabletMeta( + 1, 2, kTabletId, 15674, 4, 5, schema, 6, {{7, 8}}, UniqueId(9, 10), + TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, enable_merge_on_write)); + for (auto [start, end] : versions) { + RowsetMetaSharedPtr rs_meta(new RowsetMeta()); + init_rs_meta(rs_meta, tablet_meta, start, end, start == delete_predicate_start_version); + Status st = tablet_meta->add_rs_meta(rs_meta); + EXPECT_TRUE(st.ok()) << st; + if (!st.ok()) { + return nullptr; + } + } + auto tablet = std::make_shared(*_engine, std::move(tablet_meta), _data_dir.get()); + Status st = tablet->init(); + EXPECT_TRUE(st.ok()) << st; + if (!st.ok()) { + return nullptr; + } + auto& tablet_map = _engine->tablet_manager()->_get_tablet_map(kTabletId); + tablet_map[kTabletId] = tablet; + return tablet; + } + + // The rowset id a fixture rowset starting at `start_version` ends up with, + // matching init_rs_meta() above (numeric ids use the v1 format). + static RowsetId rowset_id_for(int64_t start_version) { + RowsetId id; + id.init(540081 + start_version); + return id; + } + + // Common scenario: a stale entry of version 50 while the query reads + // version 100 with allow_incremental set. + std::shared_ptr make_stale_decision() { + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + cache_param.__set_allow_incremental(true); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE( + QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + return runtime.get_or_make_decision(scan_ranges); + } + + // Runs two get_or_make_decision callers concurrently, parked at the + // pre-publish sync point until both built a live candidate, so the + // publish genuinely races. Returns both results (same object expected). + std::pair, + std::shared_ptr> + race_two_callers(QueryCacheRuntime& runtime, const std::vector& scan_ranges) { + auto* sp = SyncPoint::get_instance(); + std::atomic arrived {0}; + sp->set_call_back("QueryCacheRuntime::get_or_make_decision.before_publish", [&](auto&&) { + arrived.fetch_add(1); + // Deadline instead of an unbounded spin: if publication ever + // stops passing through this sync point, fail the test instead + // of hanging the whole UT binary. + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (arrived.load() < 2 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + EXPECT_GE(arrived.load(), 2) << "racer barrier timed out"; + }); + sp->enable_processing(); + + std::shared_ptr d1; + std::shared_ptr d2; + std::thread t1; + std::thread t2; + // If the second thread's constructor ever throws (resource + // exhaustion), destroying the still-joinable first one would + // std::terminate() the whole UT binary; join both on every exit. + Defer join_guard {[&] { + if (t1.joinable()) { + t1.join(); + } + if (t2.joinable()) { + t2.join(); + } + }}; + t1 = std::thread([&] { d1 = runtime.get_or_make_decision(scan_ranges); }); + t2 = std::thread([&] { d2 = runtime.get_or_make_decision(scan_ranges); }); + t1.join(); + t2.join(); + sp->disable_processing(); + sp->clear_all_call_backs(); + // Guard against vacuous passes: if the production sync point is ever + // renamed or removed, the barrier never fires and the racer tests + // would silently stop exercising the two-candidate race. + EXPECT_EQ(arrived.load(), 2); + return {d1, d2}; + } + + std::string _absolute_dir; + StorageEngine* _engine = nullptr; + std::unique_ptr _data_dir; + std::unique_ptr _cache; +}; + +TEST_F(QueryCacheIncrementalTest, incremental_success) { + create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 80}, {81, 100}}); + int64_t stale_hits_before = DorisMetrics::instance()->query_cache_stale_hit_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_stale_hit_total->value(), + stale_hits_before + 1); + + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_TRUE(decision->key_valid); + EXPECT_TRUE(decision->handle.valid()); + EXPECT_TRUE(decision->write_back_feasible); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(decision->cached_version, 50); + EXPECT_EQ(decision->cached_delta_count, 0); + EXPECT_EQ(decision->current_version, 100); + + // The pre-captured read source covers exactly the delta (50, 100]. + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 2); + EXPECT_TRUE(source->delete_predicates.empty()); + // Consumable exactly once. + EXPECT_EQ(decision->take_delta_read_source(kTabletId), nullptr); +} + +TEST_F(QueryCacheIncrementalTest, incremental_write_back_infeasible) { + create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 100}}); + // The cached entry alone (3 rows) already exceeds entry_max_rows, so the + // merged entry could never be written back: still scan only the delta, but + // announce upfront that cloning blocks for a write back is pointless. + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + cache_param.__set_allow_incremental(true); + cache_param.__set_entry_max_rows(2); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + + QueryCacheRuntime runtime(cache_param, _cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_FALSE(decision->write_back_feasible); +} + +TEST_F(QueryCacheIncrementalTest, fallback_on_agg_keys) { + create_tablet(TKeysType::AGG_KEYS, false, {{0, 50}, {51, 100}}); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + // Storage-layer aggregation breaks "cached + delta == new snapshot". + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_FALSE(decision->handle.valid()); + EXPECT_EQ(decision->incremental_fallback_reason, "keys type not append-only"); +} + +TEST_F(QueryCacheIncrementalTest, fallback_on_merge_on_read) { + // Merge-on-read UNIQUE resolves duplicates by merging across rowsets at + // read time, so a delta-only scan cannot stand alone: always fall back. + create_tablet(TKeysType::UNIQUE_KEYS, false, {{0, 50}, {51, 100}}); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "keys type not append-only"); +} + +TEST_F(QueryCacheIncrementalTest, mow_pure_append_incremental) { + // Merge-on-write UNIQUE with no delete-bitmap entry in the delta window: + // the hourly-append pattern. Incremental merge is as safe as on DUP. + create_tablet(TKeysType::UNIQUE_KEYS, true, {{0, 50}, {51, 80}, {81, 100}}); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 2); + // capture_read_source hands the delete bitmap to the delta scan, so rows + // replaced within the delta window itself are filtered by the reader. + EXPECT_NE(source->delete_bitmap, nullptr); +} + +TEST_F(QueryCacheIncrementalTest, mow_history_rewrite_falls_back) { + // A load inside the delta window marked a row of a baseline rowset as + // deleted (an upsert / backfill hit a pre-existing key): rows already + // folded into the cached entry cannot be subtracted, so fall back. + auto tablet = create_tablet(TKeysType::UNIQUE_KEYS, true, {{0, 50}, {51, 100}}); + ASSERT_NE(tablet, nullptr); + // Stamped exactly at the window's upper edge (= current_version), which + // must be inclusive. + tablet->tablet_meta()->delete_bitmap().add({rowset_id_for(0), 0, 100}, 7); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_FALSE(decision->handle.valid()); + EXPECT_EQ(decision->incremental_fallback_reason, "delta rewrites history rows"); +} + +TEST_F(QueryCacheIncrementalTest, mow_cached_side_pinned_across_classification) { + // The rewrite markers classification looks for live on the cached side, + // which the delta capture does not pin. Unpinned, a compaction spanning + // both sides can retire such a rowset without relocating the marker of a + // row it dropped, and the unused-rowset GC can then wipe that marker + // before classification reads it -- a stale entry would silently merge + // with the rows that replaced it. So the pin must cover the whole cached + // range and must still be held while classification runs, which is where + // this sync point observes it. + ASSERT_NE(create_tablet(TKeysType::UNIQUE_KEYS, true, {{0, 50}, {51, 100}}), nullptr); + auto* sp = SyncPoint::get_instance(); + int64_t pinned_start = -1; + int64_t pinned_end = -1; + int observed = 0; + sp->set_call_back("QueryCacheRuntime::_capture_tablet_delta.cached_side_pinned", + [&](auto&& args) { + auto* pinned = try_any_cast*>(args[0]); + ++observed; + EXPECT_FALSE(pinned->empty()); + if (!pinned->empty()) { + pinned_start = pinned->front()->start_version(); + pinned_end = pinned->back()->end_version(); + } + }); + sp->enable_processing(); + Defer clear_sp {[&] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + // Guard against a vacuous pass: a renamed or removed sync point would + // leave the pin unobserved while the assertions below still held. + EXPECT_EQ(observed, 1); + // The cached snapshot is everything up to the cached version, and every + // bit of it must be pinned: an unpinned tail is evidence we could lose. + EXPECT_EQ(pinned_start, 0); + EXPECT_EQ(pinned_end, 50); +} + +TEST_F(QueryCacheIncrementalTest, mow_cached_side_not_pinnable_falls_back) { + // Part of the cached snapshot is gone (compacted away and swept), so its + // rewrite markers may already have been collected: classification could no + // longer tell an append from an overwrite. The delta is unaffected and + // still captures, so only the cached-side pin can catch this. The pin asks + // quietly, which yields whatever prefix it walked -- here [0, 40], short + // of the cached version 50 -- so it is the coverage check, not emptiness, + // that must reject it. + ASSERT_NE(create_tablet(TKeysType::UNIQUE_KEYS, true, {{0, 40}, {51, 80}, {81, 100}}), nullptr); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cached versions not pinnable"); +} + +TEST_F(QueryCacheIncrementalTest, mow_irrelevant_bitmap_entries_ignored) { + // Delete-bitmap entries that cannot affect the cached snapshot must not + // spoil the incremental path: entries targeting the delta rowsets (a key + // written twice within the window, a lost sequence-column race or a delete + // sign on a window-local key), entries outside the version window (older + // dedup history, a concurrent load newer than the read version, pending + // entries at TEMP_VERSION_COMMON = 0) and empty bitmaps. + auto tablet = create_tablet(TKeysType::UNIQUE_KEYS, true, {{0, 50}, {51, 100}}); + ASSERT_NE(tablet, nullptr); + auto& delete_bitmap = tablet->tablet_meta()->delete_bitmap(); + delete_bitmap.add({rowset_id_for(51), 0, 90}, 3); // targets the delta itself + delete_bitmap.add({rowset_id_for(0), 0, 40}, 1); // version <= cached + delete_bitmap.add({rowset_id_for(0), 0, 150}, 2); // version > current + delete_bitmap.add({rowset_id_for(0), 0, 0}, 4); // pending (TEMP_VERSION_COMMON) + delete_bitmap.set({rowset_id_for(0), 0, 60}, roaring::Roaring()); // empty bitmap + // An out-of-int64-range version, only producible by corrupt persisted + // meta: must take the above-window hop and stay skipped. Placed as the + // ONLY entry of its (rowset, segment) group on purpose -- the int64-cast + // predecessor of the skip-scan classified such a key as below-window and + // its lower_bound hop then landed back on the key itself, livelocking. + // An above-window sibling in the same group would mask that: its own hop + // overshoots this key so it is never classified (an in-window sibling + // would merely relocate the landing into a two-entry livelock). + delete_bitmap.add({rowset_id_for(0), 1, std::numeric_limits::max()}, 5); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); +} + +TEST_F(QueryCacheIncrementalTest, mow_rewrite_on_later_segment_still_detected) { + // Pins two precision properties of the bitmap skip-scan: the above-window + // hop must stay within its (rowset, segment) group -- an in-window + // rewrite on a LATER segment of the same rowset must still be seen -- + // and the window's lower edge (cached_version + 1) is inclusive. A hop + // that jumped the whole rowset would miss the segment-1 evidence and + // wrongly keep the incremental path. + auto tablet = create_tablet(TKeysType::UNIQUE_KEYS, true, {{0, 50}, {51, 100}}); + ASSERT_NE(tablet, nullptr); + auto& delete_bitmap = tablet->tablet_meta()->delete_bitmap(); + delete_bitmap.add({rowset_id_for(0), 0, 150}, 1); // segment 0: above window, hops + delete_bitmap.add({rowset_id_for(0), 1, 51}, 2); // segment 1: at window begin, foreign + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "delta rewrites history rows"); +} + +TEST_F(QueryCacheIncrementalTest, fallback_on_version_gap) { + // The whole history lives in one compacted rowset [0, 100]: no version + // path can serve (50, 100] alone, so the capture comes back empty and we + // fall back. + create_tablet(TKeysType::DUP_KEYS, false, {{0, 100}}); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_EQ(decision->incremental_fallback_reason, "delta versions not capturable"); +} + +TEST_F(QueryCacheIncrementalTest, fallback_on_capture_error) { + // Non-pathfinding capture failures surface as real errors even under the + // quiet option (a pathfinding miss instead returns an empty or partial + // read source, see fallback_on_version_gap and fallback_on_partial_capture). + // The storage debug point simulates such a failure, e.g. a rowset object + // missing for a found path. + create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 100}}); + config::enable_debug_points = true; + DebugPoints::instance()->add_with_params("Tablet::capture_consistent_versions.inject_failure", + {{"tablet_id", "-2"}}); + auto decision = make_stale_decision(); + DebugPoints::instance()->remove("Tablet::capture_consistent_versions.inject_failure"); + config::enable_debug_points = false; + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_EQ(decision->incremental_fallback_reason, "delta versions not capturable"); +} + +TEST_F(QueryCacheIncrementalTest, fallback_on_delete_predicate) { + // A delete predicate inside the delta logically removes rows that are + // already folded into the cached blocks; that cannot be merged away. + create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 60}, {61, 100}}, + /*delete_predicate_start_version=*/61); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "delta contains delete predicates"); +} + +TEST_F(QueryCacheIncrementalTest, fallback_on_partial_capture) { + // The quiet capture returns whatever prefix of the version path it could + // walk: with rowsets {0,50},{51,80} and a query at version 100 (a replica + // whose local view ends short of the queried version), the (50,100] + // window walks up to 80 and stops, yielding a non-empty partial prefix. + // Treating it as INCREMENTAL would silently drop the (80,100] rows, so + // the endpoint coverage check must reject it. + create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 80}}); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_EQ(decision->incremental_fallback_reason, "delta versions not capturable"); +} + +TEST_F(QueryCacheIncrementalTest, concurrent_racers_share_one_decision) { + // The candidate decision is built outside the runtime lock (a stale + // merge-on-write decision may scan a large delete bitmap there), so two + // operators of the same instance can race: the sync point parks both + // threads right before publication, forcing two live candidates. Exactly + // one must win, both callers must observe the winner, and the metrics + // must be settled once. + create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 100}}); + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + cache_param.__set_allow_incremental(true); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + + int64_t stale_hits_before = DorisMetrics::instance()->query_cache_stale_hit_total->value(); + auto [d1, d2] = race_two_callers(runtime, scan_ranges); + + ASSERT_NE(d1, nullptr); + EXPECT_EQ(d1.get(), d2.get()); + EXPECT_EQ(d1->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_EQ(DorisMetrics::instance()->query_cache_stale_hit_total->value(), + stale_hits_before + 1); + // The winner's captured read source is intact no matter which candidate + // won: the loser's duplicate capture released with its own candidate. + auto source = d1->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 1); +} + +TEST_F(QueryCacheIncrementalTest, concurrent_racers_on_miss_settle_no_metrics) { + // No cache entry exists at all: both racers build plain-MISS candidates + // with an empty fallback reason (incremental merge never engaged), so + // publication must settle neither metric while both callers still adopt + // one shared object. + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + cache_param.__set_allow_incremental(true); + QueryCacheRuntime runtime(cache_param, _cache.get()); + + int64_t stale_before = DorisMetrics::instance()->query_cache_stale_hit_total->value(); + int64_t fallback_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto [d1, d2] = race_two_callers(runtime, scan_ranges); + + ASSERT_NE(d1, nullptr); + EXPECT_EQ(d1.get(), d2.get()); + EXPECT_EQ(d1->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(d1->incremental_fallback_reason.empty()); + EXPECT_EQ(DorisMetrics::instance()->query_cache_stale_hit_total->value(), stale_before); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallback_before); +} + +TEST_F(QueryCacheIncrementalTest, concurrent_racers_settle_fallback_once) { + // Both racers build a fallback candidate (an AGG-keys tablet rejects the + // incremental path with a non-empty reason): the fallback counter must + // move exactly once, by the published winner. + create_tablet(TKeysType::AGG_KEYS, false, {{0, 50}, {51, 100}}); + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + cache_param.__set_allow_incremental(true); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + + int64_t fallback_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto [d1, d2] = race_two_callers(runtime, scan_ranges); + + ASSERT_NE(d1, nullptr); + EXPECT_EQ(d1.get(), d2.get()); + EXPECT_EQ(d1->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d1->incremental_fallback_reason, "keys type not append-only"); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallback_before + 1); +} + } // namespace doris diff --git a/be/test/exec/scan/access_path_parser_test.cpp b/be/test/exec/scan/access_path_parser_test.cpp index d4bd6ab6c06360..02cf3013139cd3 100644 --- a/be/test/exec/scan/access_path_parser_test.cpp +++ b/be/test/exec/scan/access_path_parser_test.cpp @@ -59,11 +59,13 @@ TColumnAccessPath meta_access_path() { format::ColumnDefinition field(int32_t id, std::string name, DataTypePtr type, std::vector children = {}, - std::vector aliases = {}) { + std::vector aliases = {}, + bool has_name_mapping = false) { return { .identifier = Field::create_field(id), .name = std::move(name), .name_mapping = std::move(aliases), + .has_name_mapping = has_name_mapping, .type = std::move(type), .children = std::move(children), }; @@ -167,7 +169,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { .type = struct_type, .children = { - field(101, "a", int_type), + field(101, "a", int_type, {}, {}, true), field(205, "b", int_type, {}, {"old_b"}), }, }; @@ -199,6 +201,15 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { expect_child(column.children[0], 205, "b"); EXPECT_EQ(column.children[0].name_mapping, std::vector({"old_b"})); } + { + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "a"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); + } { auto column = root_column(100, "s", struct_type); auto status = AccessPathParser::build_nested_children( @@ -206,6 +217,7 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); expect_child(column.children[0], 101, "a"); + EXPECT_TRUE(column.children[0].has_name_mapping); expect_child(column.children[1], 205, "b"); } @@ -218,6 +230,52 @@ TEST(AccessPathParserTest, StructAccessPathMatrix) { } } +TEST(AccessPathParserTest, CurrentStructNameTakesPriorityOverHistoricalAlias) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type, int_type}, + Strings {"renamed_b", "b"}); + format::ColumnDefinition schema { + .identifier = Field::create_field(100), + .name = "s", + .type = struct_type, + .children = + { + field(1, "renamed_b", int_type, {}, {"b"}, true), + field(2, "b", int_type, {}, {}, true), + }, + }; + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "b"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 2, "b"); +} + +TEST(AccessPathParserTest, LegacyPlanRetainsOrderedStructNameAndAliasLookup) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type, int_type}, + Strings {"renamed_b", "b"}); + format::ColumnDefinition schema { + .identifier = Field::create_field(100), + .name = "s", + .type = struct_type, + .children = + { + field(1, "renamed_b", int_type, {}, {"b"}, true), + field(2, "b", int_type, {}, {}, true), + }, + }; + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "b"})}, &schema, false); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + expect_child(column.children[0], 1, "renamed_b"); +} + // Scenario: array access paths must pass through the "*" element token, then reuse struct child // parsing under the element wrapper; invalid array tokens are rejected. TEST(AccessPathParserTest, ArrayAccessPathMatrix) { @@ -235,7 +293,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { field(201, "element", element_type, { field(202, "item", string_type, {}, {"old_item"}), - field(203, "quantity", int_type), + field(203, "quantity", int_type, {}, {}, true), }), }, }; @@ -254,6 +312,18 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { EXPECT_EQ(column.children[0].children[0].name_mapping, std::vector({"old_item"})); } + { + auto column = root_column(200, "items", array_type); + auto status = AccessPathParser::build_nested_children( + &column, + std::vector {data_access_path({"items", "*", "quantity"})}, + &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + ASSERT_EQ(column.children[0].children.size(), 1); + expect_child(column.children[0].children[0], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[0].has_name_mapping); + } { auto column = root_column(200, "items", array_type); auto status = AccessPathParser::build_nested_children( @@ -264,6 +334,7 @@ TEST(AccessPathParserTest, ArrayAccessPathMatrix) { ASSERT_EQ(column.children[0].children.size(), 2); expect_child(column.children[0].children[0], 202, "item"); expect_child(column.children[0].children[1], 203, "quantity"); + EXPECT_TRUE(column.children[0].children[1].has_name_mapping); } for (const auto& invalid_path : std::vector> { @@ -293,7 +364,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { field(302, "value", value_type, { field(303, "full_name", string_type, {}, {"name"}), - field(304, "age", int_type), + field(304, "age", int_type, {}, {}, true), field(305, "gender", string_type), }), }, @@ -329,6 +400,7 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { expect_child(column.children[1], 302, "value"); ASSERT_EQ(column.children[1].children.size(), 1); expect_child(column.children[1].children[0], 304, "age"); + EXPECT_TRUE(column.children[1].children[0].has_name_mapping); } { auto column = root_column(300, "m", map_type); @@ -357,6 +429,9 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { ASSERT_TRUE(status.ok()) << status; ASSERT_EQ(column.children.size(), 2); ASSERT_EQ(column.children[1].children.size(), 3); + const auto* age = find_child_by_name(column.children[1], "age"); + ASSERT_NE(age, nullptr); + EXPECT_TRUE(age->has_name_mapping); } for (const auto& invalid_path : std::vector> { @@ -368,4 +443,21 @@ TEST(AccessPathParserTest, MapAccessPathMatrix) { } } +TEST(AccessPathParserTest, PreservesNestedInitialDefaultMetadata) { + auto binary_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {binary_type}, Strings {"data"}); + auto defaulted_child = field(101, "data", binary_type); + defaulted_child.initial_default_value = "AAEC/w=="; + defaulted_child.initial_default_value_is_base64 = true; + auto schema = field(100, "s", struct_type, {defaulted_child}); + + auto column = root_column(100, "s", struct_type); + auto status = AccessPathParser::build_nested_children( + &column, std::vector {data_access_path({"s", "data"})}, &schema); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(column.children.size(), 1); + EXPECT_EQ(column.children[0].initial_default_value, std::optional("AAEC/w==")); + EXPECT_TRUE(column.children[0].initial_default_value_is_base64); +} + } // namespace doris diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 2dc1d7f72c5570..e1a2fd0cf888f4 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -36,11 +36,15 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "exec/operator/file_scan_operator.h" +#include "exec/runtime_filter/runtime_filter_definitions.h" #include "exec/scan/file_scan_io_context.h" #include "exec/scan/file_scanner.h" #include "exec/scan/split_source_connector.h" +#include "exprs/create_predicate_function.h" #include "exprs/runtime_filter_expr.h" +#include "exprs/vbloom_predicate.h" #include "exprs/vdirect_in_predicate.h" +#include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format_v2/expr/cast.h" #include "testutil/mock/mock_runtime_state.h" @@ -95,6 +99,29 @@ TFileRangeDesc legacy_paimon_jni_range_without_reader_type() { return range; } +TEST(FileScannerTest, V1CountPushdownRequiresExplicitCountStarArguments) { + EXPECT_EQ(TPushAggOp::type::COUNT, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::COUNT, std::vector {})); + + // A missing field is an old FE plan with unknown COUNT semantics, not COUNT(*). + EXPECT_EQ(TPushAggOp::type::NONE, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::COUNT, std::nullopt)); + // V1 cannot evaluate COUNT(col) NULL/CAST semantics before replacing the reader with + // CountReader, so an explicit argument must use the normal scan path. + EXPECT_EQ(TPushAggOp::type::NONE, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::COUNT, std::vector {7})); + + // The COUNT argument field must not affect other storage-layer aggregate operations. + EXPECT_EQ(TPushAggOp::type::MINMAX, FileScanner::TEST_effective_push_down_agg_type( + TPushAggOp::type::MINMAX, std::nullopt)); +} + +TEST(FileScannerV2Test, AdaptiveBatchSizeRunsForCountFallbackOnly) { + EXPECT_TRUE(FileScannerV2::TEST_should_run_adaptive_batch_size(true, false)); + EXPECT_FALSE(FileScannerV2::TEST_should_run_adaptive_batch_size(true, true)); + EXPECT_FALSE(FileScannerV2::TEST_should_run_adaptive_batch_size(false, false)); +} + struct RetryableCloseState { int close_calls = 0; }; @@ -141,6 +168,23 @@ class UnsafePartitionPredicate final : public VExpr { const std::string _expr_name = "UnsafePartitionPredicate"; }; +class UndigestibleRuntimePredicate final : public VExpr { +public: + UndigestibleRuntimePredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + uint64_t get_digest(uint64_t) const override { return 0; } + +private: + const std::string _expr_name = "undigestible_runtime_predicate"; +}; + VExprContextSPtr runtime_filter_context(VExprSPtr impl, int filter_id) { const auto node = bool_in_pred_node(); return VExprContext::create_shared( @@ -195,6 +239,54 @@ TExprNode bool_in_pred_node() { return node; } +VExprContextSPtr int_in_runtime_filter(const std::vector& values, int filter_id) { + std::shared_ptr filter(create_set(PrimitiveType::TYPE_INT, false)); + for (const auto value : values) { + filter->insert(&value); + } + const auto node = bool_in_pred_node(); + auto impl = VDirectInPredicate::create_shared(node, std::move(filter), true); + impl->add_child(slot_ref(1, 0, std::make_shared(), "rf_key")); + return runtime_filter_context(std::move(impl), filter_id); +} + +VExprContextSPtr int_bloom_runtime_filter(const std::vector& values, int filter_id) { + std::shared_ptr filter( + create_bloom_filter(PrimitiveType::TYPE_INT, false)); + RuntimeFilterParams params; + params.filter_type = RuntimeFilterType::BLOOM_FILTER; + params.column_return_type = PrimitiveType::TYPE_INT; + params.bloom_filter_size = 1024; + filter->init_params(¶ms); + EXPECT_TRUE(filter->init_with_fixed_length(1024).ok()); + auto value_column = ColumnInt32::create(); + for (const auto value : values) { + value_column->insert_value(value); + } + ColumnPtr values_column_ptr = std::move(value_column); + filter->insert_fixed_len(values_column_ptr, 0); + + TExprNode node = bool_in_pred_node(); + node.__set_node_type(TExprNodeType::BLOOM_PRED); + node.__set_opcode(TExprOpcode::RT_FILTER); + auto impl = VBloomPredicate::create_shared(node); + impl->set_filter(std::move(filter)); + impl->add_child(slot_ref(1, 0, std::make_shared(), "rf_key")); + return runtime_filter_context(std::move(impl), filter_id); +} + +VExprContextSPtr int_minmax_runtime_filter(int32_t upper_bound, int filter_id) { + const auto int_type = std::make_shared(); + VExprSPtr impl; + TExprNode node; + EXPECT_TRUE(create_vbin_predicate(int_type, TExprOpcode::LE, impl, &node, false).ok()); + impl->add_child(slot_ref(1, 0, int_type, "rf_key")); + VExprSPtr literal; + EXPECT_TRUE(create_literal(int_type, &upper_bound, literal).ok()); + impl->add_child(std::move(literal)); + return runtime_filter_context(std::move(impl), filter_id); +} + } // namespace // Scenario: FileScannerV2::is_supported should honor table format, scan params format, and the @@ -285,6 +377,35 @@ TEST(FileScannerV2Test, IcebergPositionDeletesSupportNativeFormats) { EXPECT_FALSE(FileScannerV2::is_supported(params, avro_position_delete)); } +// Ready IN/Bloom/MinMax runtime filters all expose their payload through get_digest(). Rebuilding +// the scanner digest must therefore be stable for the same payload and isolated for a different +// payload. An RF without a complete digest remains the zero-digest safety fallback. +TEST(FileScannerV2Test, ConditionCacheDigestIncludesRuntimeFilterPayload) { + constexpr uint64_t seed = 12345; + const auto digest = [](uint64_t initial_seed, const VExprContextSPtr& conjunct) { + return Scanner::TEST_build_condition_cache_digest(initial_seed, {conjunct}); + }; + + EXPECT_EQ(digest(seed, int_in_runtime_filter({7, 9}, 1)), + digest(seed, int_in_runtime_filter({9, 7}, 1))); + EXPECT_NE(digest(seed, int_in_runtime_filter({7, 9}, 1)), + digest(seed, int_in_runtime_filter({8, 10}, 1))); + + EXPECT_EQ(digest(seed, int_bloom_runtime_filter({7, 9}, 2)), + digest(seed, int_bloom_runtime_filter({7, 9}, 2))); + EXPECT_NE(digest(seed, int_bloom_runtime_filter({7, 9}, 2)), + digest(seed, int_bloom_runtime_filter({8, 10}, 2))); + + EXPECT_EQ(digest(seed, int_minmax_runtime_filter(9, 3)), + digest(seed, int_minmax_runtime_filter(9, 3))); + EXPECT_NE(digest(seed, int_minmax_runtime_filter(9, 3)), + digest(seed, int_minmax_runtime_filter(10, 3))); + + EXPECT_EQ(digest(seed, + runtime_filter_context(std::make_shared(), 4)), + 0); +} + TEST(FileScannerV2Test, FileScanLocalStateSelectsV2ForSupportedQueriesOnly) { TQueryOptions query_options; TFileScanRangeParams params; @@ -598,6 +719,15 @@ TEST(FileScannerV2Test, FileCacheStatisticsArePublishedToScannerProfile) { EXPECT_EQ(profile.get_counter("BytesWriteIntoCache")->value(), 19); ASSERT_NE(profile.get_info_string("PeerCacheNodes"), nullptr); EXPECT_EQ(*profile.get_info_string("PeerCacheNodes"), "peer-a, peer-b"); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + ASSERT_TRUE(children.contains("IO")); + EXPECT_TRUE(children.at("IO").contains("FileCache")); } TEST(FileScannerV2Test, NotFoundIsSkippedOnlyWhenConfigured) { @@ -609,6 +739,16 @@ TEST(FileScannerV2Test, NotFoundIsSkippedOnlyWhenConfigured) { EXPECT_FALSE(FileScannerV2::TEST_should_skip_not_found(Status::OK(), true)); } +TEST(FileScannerV2Test, EndOfFileIsSkippedAsEmptySplit) { + EXPECT_TRUE(FileScannerV2::TEST_should_skip_empty(Status::EndOfFile("empty file"), false)); + // Deletion-vector and Parquet readers also use EOF to unwind an interrupted read. Once either + // scanner stop flag is visible, the same status is no longer evidence of an empty file. + EXPECT_FALSE(FileScannerV2::TEST_should_skip_empty(Status::EndOfFile("stop read."), true)); + EXPECT_FALSE( + FileScannerV2::TEST_should_skip_empty(Status::InternalError("read failed"), false)); + EXPECT_FALSE(FileScannerV2::TEST_should_skip_empty(Status::OK(), false)); +} + // Scenario: partition slots are identified from the explicit FE category when present, otherwise // from the legacy is_file_slot flag. Scanner-generated rowid columns must never be treated as // partition columns even if FE marks them as non-file slots. @@ -674,8 +814,8 @@ TEST(FileScannerV2Test, DataFileSlotClassificationMatrix) { } // Scenario: table conjuncts are cloned into global-index space before they are handed to -// TableReader. Explicit slot-id mappings use the required_slots order; missing mappings fall back -// to the slot id itself for legacy descriptors. +// TableReader. Explicit slot-id mappings use the required_slots order; missing mappings are an +// error because a scanner slot id is not a table-global ordinal. TEST(FileScannerV2Test, RewriteSlotRefsToGlobalIndexMatrix) { const auto int_type = std::make_shared(); { @@ -691,11 +831,8 @@ TEST(FileScannerV2Test, RewriteSlotRefsToGlobalIndexMatrix) { { auto expr = slot_ref(7, 99, int_type, "legacy_value"); const auto status = FileScannerV2::TEST_rewrite_slot_refs_to_global_index(&expr, {}); - ASSERT_TRUE(status.ok()) << status; - const auto* rewritten = assert_cast(expr.get()); - EXPECT_EQ(rewritten->slot_id(), 7); - EXPECT_EQ(rewritten->column_id(), 7); - EXPECT_EQ(rewritten->column_name(), "legacy_value"); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Can not resolve source slot id 7"), std::string::npos); } { auto cast_expr = format::Cast::create_shared(int_type); @@ -751,4 +888,27 @@ TEST(FileScannerTest, PartitionPruningStopsAtUnsafePredicate) { EXPECT_EQ(partition_conjuncts[0], conjuncts[0]); } +TEST(FileScannerV2Test, ScannerOwnsUnsafeConjunctAndOrderedSuffixInProfile) { + const auto bool_type = std::make_shared(); + auto unsafe_predicate = std::make_shared(); + unsafe_predicate->add_child(slot_ref(1, 0, bool_type, "part")); + VExprContextSPtrs conjuncts { + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 1), + runtime_filter_context(std::move(unsafe_predicate), 2), + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 3), + }; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("file_scanner_v2"); + FileScannerV2 scanner(&state, &profile, nullptr); + scanner.TEST_set_scanner_conjuncts(std::move(conjuncts)); + + EXPECT_EQ(scanner.TEST_table_reader_owned_conjunct_count(), 1); + EXPECT_EQ(scanner.TEST_scanner_residual_conjunct_count(), 2); + const auto* residual_predicates = profile.get_info_string("ScannerResidualPredicates"); + ASSERT_NE(residual_predicates, nullptr); + EXPECT_FALSE(residual_predicates->empty()); + EXPECT_NE(residual_predicates->find("SlotRef"), std::string::npos) << *residual_predicates; +} + } // namespace doris diff --git a/be/test/exec/scan/scanner_context_test.cpp b/be/test/exec/scan/scanner_context_test.cpp index 14a48e37a9a7a0..8be021454dd7b6 100644 --- a/be/test/exec/scan/scanner_context_test.cpp +++ b/be/test/exec/scan/scanner_context_test.cpp @@ -703,9 +703,9 @@ TEST_F(ScannerContextTest, get_free_block) { scanners, limit, scan_dependency, &shared_limit, nullptr, nullptr, 0, false, parallel_tasks); scanner_context->_newly_create_free_blocks_num = newly_create_free_blocks_num.get(); - scanner_context->_newly_create_free_blocks_num->set(0L); + scanner_context->_newly_create_free_blocks_num->set(int64_t(0)); scanner_context->_scanner_memory_used_counter = scanner_memory_used_counter.get(); - scanner_context->_scanner_memory_used_counter->set(0L); + scanner_context->_scanner_memory_used_counter->set(int64_t(0)); BlockUPtr block = scanner_context->get_free_block(/*force=*/true); ASSERT_NE(block, nullptr); ASSERT_TRUE(scanner_context->_newly_create_free_blocks_num->value() == 1); diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index 0d31b69495187e..b70f1cf9ea0a18 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -65,6 +65,23 @@ class TestScanner final : public Scanner { std::list _blocks; }; +class HighCostPredicate final : public VExpr { +public: + HighCostPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + double execute_cost() const override { return 100.0; } + +private: + const std::string _expr_name = "high_cost_stateful_predicate"; +}; + class ScannerLateArrivalRfTest : public RuntimeFilterTest { public: void SetUp() override { @@ -83,6 +100,7 @@ class ScannerLateArrivalRfTest : public RuntimeFilterTest { // the counter advances after RFs arrive and that the second call short-circuits // via the fast path at the top of the function. TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { + _runtime_states[0]->_query_options.__set_enable_adjust_conjunct_order_by_cost(true); std::vector rf_descs = { TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build(), TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build()}; @@ -104,26 +122,55 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { auto local_state = std::make_shared(_runtime_states[0].get(), op.get()); + auto initial_conjunct = VExprContext::create_shared(std::make_shared()); + ASSERT_TRUE(initial_conjunct->prepare(_runtime_states[0].get(), row_desc).ok()); + ASSERT_TRUE(initial_conjunct->open(_runtime_states[0].get()).ok()); + local_state->_conjuncts.push_back(initial_conjunct); + std::vector> rf_dependencies; ASSERT_TRUE(local_state->_helper.init(_runtime_states[0].get(), true, 0, 0, rf_dependencies, "") .ok()); + std::shared_ptr producer; + ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), rf_descs.data(), &producer).ok()); + producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + local_state->_helper._consumers[0]->signal(producer.get()); + ASSERT_TRUE(local_state->_helper + .acquire_runtime_filter(_runtime_states[0].get(), local_state->_conjuncts, + row_desc) + .ok()); + ASSERT_EQ(local_state->_conjuncts.size(), 2); + auto scanner = std::make_unique(_runtime_states[0].get(), local_state.get(), -1 /*limit*/, &_profile); - ASSERT_TRUE(scanner->init(_runtime_states[0].get(), {}).ok()); + ASSERT_TRUE(scanner->init(_runtime_states[0].get(), local_state->_conjuncts).ok()); + auto second_scanner = std::make_unique(_runtime_states[0].get(), local_state.get(), + -1 /*limit*/, &_profile); + ASSERT_TRUE(second_scanner->init(_runtime_states[0].get(), local_state->_conjuncts).ok()); ASSERT_EQ(scanner->_total_rf_num, 2); ASSERT_EQ(scanner->_applied_rf_num, 0); - std::shared_ptr producer; - ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), rf_descs.data(), &producer).ok()); - producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); - local_state->_helper._consumers[0]->signal(producer.get()); local_state->_helper._consumers[1]->signal(producer.get()); // First call after both RFs arrived: counter must advance to total. Before // the fix this stayed at 0 because the assignment was missing. ASSERT_TRUE(scanner->try_append_late_arrival_runtime_filter().ok()); ASSERT_EQ(scanner->_applied_rf_num, 2); + ASSERT_EQ(scanner->_late_arrival_rf_conjuncts.size(), 1); + for (const auto& conjunct : scanner->_late_arrival_rf_conjuncts) { + EXPECT_NE(dynamic_cast(conjunct->root().get()), nullptr); + } + ASSERT_EQ(scanner->_conjuncts.size(), 3); + EXPECT_EQ(scanner->_conjuncts.back()->expr_name(), "high_cost_stateful_predicate"); + + // The first scanner consumes the shared helper's expression, so another scanner can only get + // the exact delta from the local state's append-only RF batch history. + ASSERT_TRUE(second_scanner->try_append_late_arrival_runtime_filter().ok()); + ASSERT_EQ(second_scanner->_applied_rf_num, 2); + ASSERT_EQ(second_scanner->_late_arrival_rf_conjuncts.size(), 1); + EXPECT_NE(dynamic_cast( + second_scanner->_late_arrival_rf_conjuncts[0]->root().get()), + nullptr); // Second call: must hit the fast-path early return without re-cloning. // We clear `_conjuncts` and verify the function does NOT repopulate them; diff --git a/be/test/exec/sink/viceberg_delete_sink_test.cpp b/be/test/exec/sink/viceberg_delete_sink_test.cpp index d9fc5086503d90..7faa77ed702c68 100644 --- a/be/test/exec/sink/viceberg_delete_sink_test.cpp +++ b/be/test/exec/sink/viceberg_delete_sink_test.cpp @@ -33,6 +33,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "exec/common/endian.h" +#include "format/table/deletion_vector.h" #include "gen_cpp/DataSinks_types.h" #include "gen_cpp/Types_types.h" #include "runtime/runtime_state.h" @@ -507,6 +508,19 @@ TEST_F(VIcebergDeleteSinkTest, TestUnsupportedDeleteType) { ASSERT_FALSE(status.ok()); } +TEST_F(VIcebergDeleteSinkTest, TestValidateDeletionVectorContentSize) { + constexpr size_t max_bitmap_size = static_cast(MAX_ICEBERG_DELETION_VECTOR_BYTES) - + ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES; + int64_t content_size = 0; + ASSERT_TRUE( + calculate_iceberg_deletion_vector_content_size(max_bitmap_size, &content_size).ok()); + ASSERT_EQ(MAX_ICEBERG_DELETION_VECTOR_BYTES, content_size); + + const auto unsupported_status = + calculate_iceberg_deletion_vector_content_size(max_bitmap_size + 1, &content_size); + ASSERT_TRUE(unsupported_status.is()) << unsupported_status; +} + TEST_F(VIcebergDeleteSinkTest, TestWriteDeletionVectorsToSingleSharedPuffin) { std::filesystem::path temp_dir = std::filesystem::temp_directory_path() / ("iceberg_delete_sink_test_" + generate_uuid_string()); diff --git a/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp new file mode 100644 index 00000000000000..d453177cf25044 --- /dev/null +++ b/be/test/exec/sink/writer/iceberg/iceberg_partition_writer_test.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +#include "exec/sink/writer/iceberg/viceberg_partition_writer.h" + +namespace doris { + +namespace { + +class FakeFileFormatTransformer final : public VFileFormatTransformer { +public: + explicit FakeFileFormatTransformer(const VExprContextSPtrs& output_exprs) + : VFileFormatTransformer(nullptr, output_exprs, false) {} + + Status open() override { return Status::OK(); } + Status write(const Block&) override { return Status::OK(); } + Status close() override { return Status::OK(); } + int64_t written_len() override { return 64; } +}; + +TDataSink make_table_sink(std::optional collect_column_stats) { + TIcebergTableSink iceberg_sink; + if (collect_column_stats.has_value()) { + iceberg_sink.__set_collect_column_stats(*collect_column_stats); + } + TDataSink sink; + sink.__set_type(TDataSinkType::ICEBERG_TABLE_SINK); + sink.__set_iceberg_table_sink(iceberg_sink); + return sink; +} + +} // namespace + +class VIcebergPartitionWriterTest : public testing::Test { +protected: + static std::unique_ptr make_writer( + const TDataSink& sink, const VExprContextSPtrs& output_exprs, + const iceberg::Schema& schema, const std::string* schema_json, + const std::map& hadoop_conf) { + IPartitionWriterBase::WriteInfo write_info; + write_info.file_type = TFileType::FILE_LOCAL; + return std::make_unique( + sink, std::vector {}, output_exprs, schema, schema_json, + std::vector {}, std::move(write_info), "data", 0, + TFileFormatType::FORMAT_ORC, TFileCompressType::ZLIB, hadoop_conf); + } + + static void install_fake_transformer(VIcebergPartitionWriter* writer, + const VExprContextSPtrs& output_exprs) { + writer->_file_format_transformer = + std::make_unique(output_exprs); + } + + static Status build_commit_data(VIcebergPartitionWriter* writer, + TIcebergCommitData* commit_data) { + return writer->_build_iceberg_commit_data(commit_data); + } + + static bool collect_column_stats(const VIcebergPartitionWriter& writer) { + return writer._collect_column_stats; + } +}; + +TEST_F(VIcebergPartitionWriterTest, OrcSkipsFooterCollectionWhenMetricsAreDisabled) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto writer = + make_writer(make_table_sink(false), output_exprs, schema, &schema_json, hadoop_conf); + install_fake_transformer(writer.get(), output_exprs); + + TIcebergCommitData commit_data; + ASSERT_TRUE(build_commit_data(writer.get(), &commit_data).ok()); + EXPECT_FALSE(commit_data.__isset.column_stats); +} + +TEST_F(VIcebergPartitionWriterTest, MissingPolicyKeepsCollectionEnabledForRollingUpgrade) { + VExprContextSPtrs output_exprs; + iceberg::Schema schema(std::vector {}); + std::string schema_json; + std::map hadoop_conf; + auto writer = make_writer(make_table_sink(std::nullopt), output_exprs, schema, &schema_json, + hadoop_conf); + + EXPECT_TRUE(collect_column_stats(*writer)); +} + +} // namespace doris diff --git a/be/test/exprs/bloom_filter_func_test.cpp b/be/test/exprs/bloom_filter_func_test.cpp index bc0ce276a37e7e..5e0c02479be719 100644 --- a/be/test/exprs/bloom_filter_func_test.cpp +++ b/be/test/exprs/bloom_filter_func_test.cpp @@ -29,9 +29,12 @@ #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" #include "core/value/vdatetime_value.h" +#include "exprs/block_bloom_filter.hpp" #include "exprs/create_predicate_function.h" #include "exprs/function/cast/cast_to_datev2_impl.hpp" #include "gtest/gtest.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/thread_context.h" #include "testutil/column_helper.h" #include "util/url_coding.h" @@ -79,6 +82,30 @@ TEST_F(BloomFilterFuncTest, Init) { bloom_filter_func2.light_copy(&bloom_filter_func); } +TEST_F(BloomFilterFuncTest, TrackBlockBloomFilterMemory) { + constexpr int initial_log_space_bytes = 22; + constexpr int resized_log_space_bytes = 23; + auto mem_tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, + "BlockBloomFilterMemoryTest"); + auto switch_mem_tracker = SwitchThreadMemTrackerLimiter(mem_tracker); + thread_context()->thread_mem_tracker_mgr->flush_untracked_mem(); + const int64_t initial_consumption = mem_tracker->consumption(); + + BlockBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter.init(initial_log_space_bytes, 0).ok()); + thread_context()->thread_mem_tracker_mgr->flush_untracked_mem(); + EXPECT_EQ(mem_tracker->consumption(), initial_consumption + (1ULL << initial_log_space_bytes)); + EXPECT_EQ(reinterpret_cast(bloom_filter.directory().data) % 32, 0); + + ASSERT_TRUE(bloom_filter.init(resized_log_space_bytes, 0).ok()); + thread_context()->thread_mem_tracker_mgr->flush_untracked_mem(); + EXPECT_EQ(mem_tracker->consumption(), initial_consumption + (1ULL << resized_log_space_bytes)); + + bloom_filter.close(); + thread_context()->thread_mem_tracker_mgr->flush_untracked_mem(); + EXPECT_EQ(mem_tracker->consumption(), initial_consumption); +} + TEST_F(BloomFilterFuncTest, FixedLenToUInt32) { fixed_len_to_uint32_v2 fixed_lenv2; { diff --git a/be/test/exprs/function/cast/cast_to_array_test.cpp b/be/test/exprs/function/cast/cast_to_array_test.cpp index 3ac5ac4476da38..bf3b8a95b1633a 100644 --- a/be/test/exprs/function/cast/cast_to_array_test.cpp +++ b/be/test/exprs/function/cast/cast_to_array_test.cpp @@ -21,6 +21,7 @@ #include #include "core/column/column_array.h" +#include "core/data_type/data_type_array.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" @@ -196,4 +197,94 @@ TEST_F(FunctionCastTest, test_from_string_to_array_bool) { builder.build(), false); } +TEST_F(FunctionCastTest, test_from_null_literal_array_to_nested_array) { + auto null_literal_type = std::make_shared(); + null_literal_type->set_null_literal(true); + auto from_nested_type = std::make_shared(null_literal_type); + auto from_type = std::make_shared(from_nested_type); + + auto from_nested_column = + ColumnHelper::create_nullable_column({0, 0, 0}, {true, true, true}); + auto from_offsets = ColumnHelper::create_column_offsets({0, 1, 3}); + auto from_column = ColumnArray::create(std::move(from_nested_column), std::move(from_offsets)); + + auto to_type = std::make_shared( + std::make_shared(std::make_shared())); + + auto ctx = create_context(false); + auto fn = get_cast_wrapper(ctx.get(), from_type, to_type); + ASSERT_TRUE(fn != nullptr); + + Block block = { + {std::move(from_column), from_type, "from"}, + {nullptr, to_type, "to"}, + }; + ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr)); + + const auto* result_array = + check_and_get_column(block.get_by_position(1).column.get()); + ASSERT_NE(result_array, nullptr); + EXPECT_EQ(result_array->get_offsets(), ColumnArray::Offsets64({0, 1, 3})); + + const auto* result_nested_nullable = + check_and_get_column(&result_array->get_data()); + ASSERT_NE(result_nested_nullable, nullptr); + EXPECT_EQ(result_nested_nullable->get_null_map_data(), NullMap({1, 1, 1})); + + const auto* result_nested_array = + check_and_get_column(&result_nested_nullable->get_nested_column()); + ASSERT_NE(result_nested_array, nullptr); + EXPECT_EQ(result_nested_array->size(), 3); + EXPECT_EQ(result_nested_array->get_offsets(), ColumnArray::Offsets64({0, 0, 0})); +} + +TEST_F(FunctionCastTest, test_from_nested_null_literal_array_to_deeper_array) { + auto null_literal_type = std::make_shared(); + null_literal_type->set_null_literal(true); + auto from_inner_type = std::make_shared(null_literal_type); + auto from_type = std::make_shared(from_inner_type); + + auto leaf_column = ColumnHelper::create_nullable_column({0}, {true}); + auto inner_offsets = ColumnHelper::create_column_offsets({1}); + auto inner_column = ColumnArray::create(std::move(leaf_column), std::move(inner_offsets)); + auto inner_null_map = ColumnHelper::create_column({false}); + auto nullable_inner_column = + ColumnNullable::create(std::move(inner_column), std::move(inner_null_map)); + auto outer_offsets = ColumnHelper::create_column_offsets({1}); + auto from_column = + ColumnArray::create(std::move(nullable_inner_column), std::move(outer_offsets)); + + auto to_type = std::make_shared(std::make_shared( + std::make_shared(std::make_shared()))); + + auto ctx = create_context(false); + auto fn = get_cast_wrapper(ctx.get(), from_type, to_type); + ASSERT_TRUE(fn != nullptr); + + Block block = { + {std::move(from_column), from_type, "from"}, + {nullptr, to_type, "to"}, + }; + ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr)); + EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 0), "[[null]]"); +} + +TEST_F(FunctionCastTest, test_from_non_null_array_to_nested_array_is_rejected) { + ColumnArrayBuilder builder; + builder.add({1}); + auto from = builder.build(); + auto to_type = std::make_shared( + std::make_shared(std::make_shared())); + + auto ctx = create_context(false); + auto fn = get_cast_wrapper(ctx.get(), from.type, to_type); + ASSERT_TRUE(fn != nullptr); + + Block block = { + from, + {nullptr, to_type, "to"}, + }; + EXPECT_FALSE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr)); +} + } // namespace doris diff --git a/be/test/exprs/function/cast/cast_to_string_api_test.cpp b/be/test/exprs/function/cast/cast_to_string_api_test.cpp index b8bd0c1255bee8..8590f938e5d6c0 100644 --- a/be/test/exprs/function/cast/cast_to_string_api_test.cpp +++ b/be/test/exprs/function/cast/cast_to_string_api_test.cpp @@ -69,22 +69,22 @@ TEST(CastToStringTest, test) { EXPECT_EQ(str, "1234567.89"); } { - Decimal64 num = -123456789012345678; + Decimal64 num = int64_t(-123456789012345678); std::string str = CastToString::from_decimal(num, 4); EXPECT_EQ(str, "-12345678901234.5678"); } { - Decimal128V2 num = 1234567890123; + Decimal128V2 num = int64_t(1234567890123); std::string str = CastToString::from_decimal(num, 6); EXPECT_EQ(str, "1234.567890"); } { - Decimal128V3 num = 1234567890567890; + Decimal128V3 num = int64_t(1234567890567890); std::string str = CastToString::from_decimal(num, 8); EXPECT_EQ(str, "12345678.90567890"); } { - Decimal256 num {1234567890567890}; + Decimal256 num {int64_t(1234567890567890)}; std::string str = CastToString::from_decimal(num, 10); EXPECT_EQ(str, "123456.7890567890"); } diff --git a/be/test/exprs/lambda_function/array_map_function_test.cpp b/be/test/exprs/lambda_function/array_map_function_test.cpp new file mode 100644 index 00000000000000..42469b52c3bc04 --- /dev/null +++ b/be/test/exprs/lambda_function/array_map_function_test.cpp @@ -0,0 +1,1409 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "exprs/vcolumn_ref.h" +#include "exprs/vexpr_context.h" +#include "exprs/vlambda_function_call_expr.h" +#include "exprs/vlambda_function_expr.h" +#include "exprs/vslot_ref.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" + +namespace doris { + +class MockColumnExpr final : public VExpr { +public: + MockColumnExpr(ColumnPtr column, DataTypePtr type, std::string name) + : VExpr(type, false), + _column(std::move(column)), + _type(std::move(type)), + _name(std::move(name)) {} + + const std::string& expr_name() const override { return _name; } + + Status execute_column_impl(VExprContext* /*context*/, const Block* /*block*/, + const Selector* /*selector*/, size_t /*count*/, + ColumnPtr& result_column) const override { + result_column = _column; + return Status::OK(); + } + + DataTypePtr execute_type(const Block* /*block*/) const override { return _type; } + +private: + ColumnPtr _column; + DataTypePtr _type; + std::string _name; +}; + +class MockConstColumnExpr final : public VExpr { +public: + MockConstColumnExpr(ColumnPtr column, DataTypePtr type, std::string name) + : VExpr(type, false), + _column(std::move(column)), + _type(std::move(type)), + _name(std::move(name)) {} + + const std::string& expr_name() const override { return _name; } + + Status execute_column_impl(VExprContext* /*context*/, const Block* /*block*/, + const Selector* /*selector*/, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnConst::create(_column, count); + return Status::OK(); + } + + DataTypePtr execute_type(const Block* /*block*/) const override { return _type; } + +private: + ColumnPtr _column; + DataTypePtr _type; + std::string _name; +}; + +class MockBodyExpr final : public VExpr { +public: + MockBodyExpr(DataTypePtr type, std::string name) + : VExpr(type, false), _type(std::move(type)), _name(std::move(name)) {} + + const std::string& expr_name() const override { return _name; } + + Status execute_column_impl(VExprContext* /*context*/, const Block* /*block*/, + const Selector* /*selector*/, size_t /*count*/, + ColumnPtr& /*result_column*/) const override { + return Status::InternalError("mock body should not be executed"); + } + + DataTypePtr execute_type(const Block* /*block*/) const override { return _type; } + +private: + DataTypePtr _type; + std::string _name; +}; + +class MockSubtractExpr final : public VExpr { +public: + explicit MockSubtractExpr(DataTypePtr type) : VExpr(type, false), _type(std::move(type)) {} + + const std::string& expr_name() const override { return _name; } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + ColumnPtr left; + ColumnPtr right; + RETURN_IF_ERROR(get_child(0)->execute_column(context, block, selector, count, left)); + RETURN_IF_ERROR(get_child(1)->execute_column(context, block, selector, count, right)); + left = left->convert_to_full_column_if_const(); + right = right->convert_to_full_column_if_const(); + + const auto& left_data = _get_int_data(left); + const auto& right_data = _get_int_data(right); + auto result = ColumnInt32::create(); + for (size_t i = 0; i < count; ++i) { + result->insert_value(left_data.get_element(i) - right_data.get_element(i)); + } + result_column = std::move(result); + return Status::OK(); + } + + DataTypePtr execute_type(const Block* /*block*/) const override { return _type; } + +private: + const ColumnInt32& _get_int_data(const ColumnPtr& column) const { + if (const auto* nullable = check_and_get_column(column.get())) { + return assert_cast(nullable->get_nested_column()); + } + return assert_cast(*column); + } + + DataTypePtr _type; + std::string _name = "mock_subtract"; +}; + +class MockAddExpr final : public VExpr { +public: + explicit MockAddExpr(DataTypePtr type) : VExpr(type, false), _type(std::move(type)) {} + + const std::string& expr_name() const override { return _name; } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + ColumnPtr left; + ColumnPtr right; + RETURN_IF_ERROR(get_child(0)->execute_column(context, block, selector, count, left)); + RETURN_IF_ERROR(get_child(1)->execute_column(context, block, selector, count, right)); + left = left->convert_to_full_column_if_const(); + right = right->convert_to_full_column_if_const(); + + const auto& left_data = _get_int_data(left); + const auto& right_data = _get_int_data(right); + auto result = ColumnInt32::create(); + for (size_t i = 0; i < count; ++i) { + result->insert_value(left_data.get_element(i) + right_data.get_element(i)); + } + result_column = std::move(result); + return Status::OK(); + } + + DataTypePtr execute_type(const Block* /*block*/) const override { return _type; } + +private: + const ColumnInt32& _get_int_data(const ColumnPtr& column) const { + if (const auto* nullable = check_and_get_column(column.get())) { + return assert_cast(nullable->get_nested_column()); + } + return assert_cast(*column); + } + + DataTypePtr _type; + std::string _name = "mock_add"; +}; + +class MockMultiplyExpr final : public VExpr { +public: + explicit MockMultiplyExpr(DataTypePtr type) : VExpr(type, false), _type(std::move(type)) {} + + const std::string& expr_name() const override { return _name; } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + ColumnPtr left; + ColumnPtr right; + RETURN_IF_ERROR(get_child(0)->execute_column(context, block, selector, count, left)); + RETURN_IF_ERROR(get_child(1)->execute_column(context, block, selector, count, right)); + left = left->convert_to_full_column_if_const(); + right = right->convert_to_full_column_if_const(); + + const auto& left_data = _get_int_data(left); + const auto& right_data = _get_int_data(right); + auto result = ColumnInt32::create(); + for (size_t i = 0; i < count; ++i) { + result->insert_value(left_data.get_element(i) * right_data.get_element(i)); + } + result_column = std::move(result); + return Status::OK(); + } + + DataTypePtr execute_type(const Block* /*block*/) const override { return _type; } + +private: + const ColumnInt32& _get_int_data(const ColumnPtr& column) const { + if (const auto* nullable = check_and_get_column(column.get())) { + return assert_cast(nullable->get_nested_column()); + } + return assert_cast(*column); + } + + DataTypePtr _type; + std::string _name = "mock_multiply"; +}; + +class MockCompareExpr final : public VExpr { +public: + explicit MockCompareExpr(DataTypePtr type) : VExpr(type, false), _type(std::move(type)) {} + + const std::string& expr_name() const override { return _name; } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + ColumnPtr left; + ColumnPtr right; + RETURN_IF_ERROR(get_child(0)->execute_column(context, block, selector, count, left)); + RETURN_IF_ERROR(get_child(1)->execute_column(context, block, selector, count, right)); + left = left->convert_to_full_column_if_const(); + right = right->convert_to_full_column_if_const(); + + const auto& left_data = _get_int_data(left); + const auto& right_data = _get_int_data(right); + auto result = ColumnInt8::create(); + for (size_t i = 0; i < count; ++i) { + const auto left_value = left_data.get_element(i); + const auto right_value = right_data.get_element(i); + int8_t compare_result = 0; + if (left_value < right_value) { + compare_result = -1; + } else if (left_value > right_value) { + compare_result = 1; + } + result->insert_value(compare_result); + } + result_column = std::move(result); + return Status::OK(); + } + + DataTypePtr execute_type(const Block* /*block*/) const override { return _type; } + +private: + const ColumnInt32& _get_int_data(const ColumnPtr& column) const { + if (const auto* nullable = check_and_get_column(column.get())) { + return assert_cast(nullable->get_nested_column()); + } + return assert_cast(*column); + } + + DataTypePtr _type; + std::string _name = "mock_compare"; +}; + +static TExprNode make_lambda_call_node(const DataTypePtr& type, int num_children, + const std::string& function_name = "array_map") { + TExprNode node; + node.__set_node_type(TExprNodeType::LAMBDA_FUNCTION_CALL_EXPR); + node.__set_num_children(num_children); + node.__set_type(type->to_thrift()); + node.__set_is_nullable(type->is_nullable()); + + TFunction fn; + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + fn.__set_name(fn_name); + node.__set_fn(fn); + return node; +} + +static TExprNode make_lambda_expr_node(const DataTypePtr& type, + const std::vector& argument_names, + bool set_argument_names = true) { + TExprNode node; + node.__set_node_type(TExprNodeType::LAMBDA_FUNCTION_EXPR); + node.__set_num_children(1); + node.__set_type(type->to_thrift()); + node.__set_is_nullable(type->is_nullable()); + if (set_argument_names) { + node.__set_lambda_argument_names(argument_names); + } + return node; +} + +static TExprNode make_column_ref_node(int column_id, const std::string& column_name, + const DataTypePtr& type) { + TExprNode node; + node.__set_node_type(TExprNodeType::COLUMN_REF); + node.__set_num_children(0); + node.__set_type(type->to_thrift()); + node.__set_is_nullable(type->is_nullable()); + + TColumnRef column_ref; + column_ref.__set_column_id(column_id); + column_ref.__set_column_name(column_name); + node.__set_column_ref(column_ref); + return node; +} + +static VExprSPtr make_slot_ref(int column_id, const std::string& column_name, + const DataTypePtr& type) { + static std::vector> column_names; + column_names.push_back(std::make_unique(column_name)); + + auto ref = VSlotRef::create_shared(); + ref->set_node_type(TExprNodeType::SLOT_REF); + ref->set_slot_id(-1); + ref->set_column_id(column_id); + ref->set_column_name(column_names.back().get()); + ref->data_type() = type; + return ref; +} + +static ColumnPtr make_int_column(const std::vector& values) { + auto column = ColumnInt32::create(); + for (auto value : values) { + column->insert_value(value); + } + return column; +} + +static ColumnPtr make_int_array_column(const std::vector>& rows) { + auto int_column = ColumnInt32::create(); + auto offsets = ColumnArray::ColumnOffsets::create(); + int64_t offset = 0; + for (const auto& row : rows) { + for (auto value : row) { + int_column->insert_value(value); + } + offset += row.size(); + offsets->insert_value(offset); + } + auto int_null_map = ColumnUInt8::create(int_column->size(), 0); + auto nullable_int_column = + ColumnNullable::create(std::move(int_column), std::move(int_null_map)); + return ColumnArray::create(std::move(nullable_int_column), std::move(offsets)); +} + +static ColumnPtr make_nested_int_array_column() { + // Two input rows: + // row 0: [[1, 2], [3]] + // row 1: [[4, 5]] + auto int_column = ColumnInt32::create(); + for (int32_t value : {1, 2, 3, 4, 5}) { + int_column->insert_value(value); + } + auto int_null_map = ColumnUInt8::create(int_column->size(), 0); + auto nullable_int_column = + ColumnNullable::create(std::move(int_column), std::move(int_null_map)); + + auto inner_offsets = ColumnArray::ColumnOffsets::create(); + for (int64_t offset : {2, 3, 5}) { + inner_offsets->insert_value(offset); + } + auto inner_array_column = + ColumnArray::create(std::move(nullable_int_column), std::move(inner_offsets)); + auto inner_array_null_map = ColumnUInt8::create(inner_array_column->size(), 0); + auto nullable_inner_array_column = + ColumnNullable::create(std::move(inner_array_column), std::move(inner_array_null_map)); + + auto outer_offsets = ColumnArray::ColumnOffsets::create(); + for (int64_t offset : {2, 3}) { + outer_offsets->insert_value(offset); + } + return ColumnArray::create(std::move(nullable_inner_array_column), std::move(outer_offsets)); +} + +static ColumnPtr make_nested_unsorted_int_array_column() { + // Two input rows: + // row 0: [[2, 1], [3]] + // row 1: [[5, 4]] + auto int_column = ColumnInt32::create(); + for (int32_t value : {2, 1, 3, 5, 4}) { + int_column->insert_value(value); + } + auto int_null_map = ColumnUInt8::create(int_column->size(), 0); + auto nullable_int_column = + ColumnNullable::create(std::move(int_column), std::move(int_null_map)); + + auto inner_offsets = ColumnArray::ColumnOffsets::create(); + for (int64_t offset : {2, 3, 5}) { + inner_offsets->insert_value(offset); + } + auto inner_array_column = + ColumnArray::create(std::move(nullable_int_column), std::move(inner_offsets)); + auto inner_array_null_map = ColumnUInt8::create(inner_array_column->size(), 0); + auto nullable_inner_array_column = + ColumnNullable::create(std::move(inner_array_column), std::move(inner_array_null_map)); + + auto outer_offsets = ColumnArray::ColumnOffsets::create(); + for (int64_t offset : {2, 3}) { + outer_offsets->insert_value(offset); + } + return ColumnArray::create(std::move(nullable_inner_array_column), std::move(outer_offsets)); +} + +static void open_expr(const VExprSPtr& expr, VExprContext* context) { + RuntimeState state; + RowDescriptor row_desc; + ASSERT_TRUE(expr->prepare(&state, row_desc, context).ok()); + ASSERT_TRUE(expr->open(&state, context, FunctionContext::THREAD_LOCAL).ok()); +} + +TEST(ArrayMapFunctionTest, NestedLambdaWithSameArgumentNameUsesInnerScope) { + auto int_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"x"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"x"})); + + // This mirrors array_map(x -> array_map(x -> x, x), nested_array). Both + // lambda arguments are named "x"; only their scopes and element types are + // different. The non-ordinal column ids cover FE plans where a lambda + // argument ColumnRef id is not the same as its argument position. + auto inner_x = VColumnRef::create_shared(make_column_ref_node(2, "x", int_type)); + auto outer_x = VColumnRef::create_shared(make_column_ref_node(1, "x", array_int_type)); + + inner_lambda->add_child(inner_x); + inner_call->add_child(inner_lambda); + inner_call->add_child(outer_x); + outer_lambda->add_child(inner_call); + root->add_child(outer_lambda); + root->add_child(std::make_shared(make_nested_int_array_column(), + nested_array_int_type, "nested_array")); + + VExprContext context(root); + open_expr(root, &context); + + // Keep one ordinary input column before the array_map argument. The test + // verifies nested lambda gap calculation when lambda blocks need to preserve + // existing input columns as well as append current lambda arguments. + Block block; + block.insert({make_int_column({10, 20}), int_type, "ordinary_input"}); + + ColumnPtr result; + auto status = root->execute_column(&context, &block, nullptr, block.rows(), result); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& outer_array = assert_cast(*result); + ASSERT_EQ(outer_array.size(), 2); + ASSERT_EQ(outer_array.get_offsets()[0], 2); + ASSERT_EQ(outer_array.get_offsets()[1], 3); + + const auto* nullable_inner_arrays = + check_and_get_column(&outer_array.get_data()); + ASSERT_NE(nullable_inner_arrays, nullptr); + for (size_t i = 0; i < nullable_inner_arrays->size(); ++i) { + EXPECT_FALSE(nullable_inner_arrays->is_null_at(i)); + } + + const auto& inner_arrays = + assert_cast(nullable_inner_arrays->get_nested_column()); + ASSERT_EQ(inner_arrays.size(), 3); + ASSERT_EQ(inner_arrays.get_offsets()[0], 2); + ASSERT_EQ(inner_arrays.get_offsets()[1], 3); + ASSERT_EQ(inner_arrays.get_offsets()[2], 5); + + const auto* nullable_values = check_and_get_column(&inner_arrays.get_data()); + ASSERT_NE(nullable_values, nullptr); + for (size_t i = 0; i < nullable_values->size(); ++i) { + EXPECT_FALSE(nullable_values->is_null_at(i)); + } + + const auto& values = assert_cast(nullable_values->get_nested_column()); + ASSERT_EQ(values.size(), 5); + EXPECT_EQ(values.get_element(0), 1); + EXPECT_EQ(values.get_element(1), 2); + EXPECT_EQ(values.get_element(2), 3); + EXPECT_EQ(values.get_element(3), 4); + EXPECT_EQ(values.get_element(4), 5); +} + +TEST(ArrayMapFunctionTest, NamedLambdaWithFewerArgumentsThanArraysUsesDeclaredBindings) { + auto int_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + + auto root = VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 3)); + auto lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"x"})); + + // This mirrors a BE-compatible plan shape like array_map(x -> x, arr1, arr2). + // Only x is part of the lambda binding frame; the extra array input is still + // materialized for size/offset validation but must not require a lambda name. + auto x = VColumnRef::create_shared(make_column_ref_node(5, "x", int_type)); + + lambda->add_child(x); + root->add_child(lambda); + root->add_child(std::make_shared(make_int_array_column({{10, 20}, {30}}), + array_int_type, "arr1")); + root->add_child(std::make_shared(make_int_array_column({{1, 2}, {3}}), + array_int_type, "arr2")); + + VExprContext context(root); + open_expr(root, &context); + + Block block; + ColumnPtr result; + auto status = root->execute_column(&context, &block, nullptr, 2, result); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& result_array = assert_cast(*result); + ASSERT_EQ(result_array.size(), 2); + ASSERT_EQ(result_array.get_offsets()[0], 2); + ASSERT_EQ(result_array.get_offsets()[1], 3); + const auto* nullable_values = check_and_get_column(&result_array.get_data()); + ASSERT_NE(nullable_values, nullptr); + const auto& values = assert_cast(nullable_values->get_nested_column()); + ASSERT_EQ(values.size(), 3); + EXPECT_EQ(values.get_element(0), 10); + EXPECT_EQ(values.get_element(1), 20); + EXPECT_EQ(values.get_element(2), 30); +} + +TEST(ArrayMapFunctionTest, NestedArraySortInsideArrayMapSkipsArrayMapArgumentInference) { + auto int_type = std::make_shared(); + auto int8_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nullable_array_int_type = std::make_shared(array_int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"a"})); + auto sort_call = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(nullable_array_int_type, 2, "array_sort")); + auto sort_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"a", "b"})); + auto compare = std::make_shared(int8_type); + + // This mirrors array_map(a -> array_sort((a, b) -> a - b, a), nested_array). + // FE represents the second comparator argument by cloning the first + // ColumnRef and only changing column_id to 1, so both comparator ColumnRefs + // can have the same name. array_sort must keep its comparator arguments + // position-based instead of using array_map's name-based binding. + auto sort_a = VColumnRef::create_shared(make_column_ref_node(0, "a", int_type)); + auto sort_b = VColumnRef::create_shared(make_column_ref_node(1, "a", int_type)); + auto outer_a = VColumnRef::create_shared(make_column_ref_node(1, "a", array_int_type)); + + compare->add_child(sort_a); + compare->add_child(sort_b); + sort_lambda->add_child(compare); + sort_call->add_child(sort_lambda); + sort_call->add_child(outer_a); + outer_lambda->add_child(sort_call); + root->add_child(outer_lambda); + root->add_child(std::make_shared(make_nested_unsorted_int_array_column(), + nested_array_int_type, "nested_array")); + + VExprContext context(root); + open_expr(root, &context); + + Block block; + block.insert({make_int_column({10, 20}), int_type, "ordinary_input"}); + + ColumnPtr result; + auto status = root->execute_column(&context, &block, nullptr, block.rows(), result); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& outer_array = assert_cast(*result); + ASSERT_EQ(outer_array.size(), 2); + ASSERT_EQ(outer_array.get_offsets()[0], 2); + ASSERT_EQ(outer_array.get_offsets()[1], 3); + + const auto* nullable_inner_arrays = + check_and_get_column(&outer_array.get_data()); + ASSERT_NE(nullable_inner_arrays, nullptr); + + const auto& inner_arrays = + assert_cast(nullable_inner_arrays->get_nested_column()); + ASSERT_EQ(inner_arrays.size(), 3); + ASSERT_EQ(inner_arrays.get_offsets()[0], 2); + ASSERT_EQ(inner_arrays.get_offsets()[1], 3); + ASSERT_EQ(inner_arrays.get_offsets()[2], 5); + + const auto* nullable_values = check_and_get_column(&inner_arrays.get_data()); + ASSERT_NE(nullable_values, nullptr); + const auto& values = assert_cast(nullable_values->get_nested_column()); + ASSERT_EQ(values.size(), 5); + EXPECT_EQ(values.get_element(0), 1); + EXPECT_EQ(values.get_element(1), 2); + EXPECT_EQ(values.get_element(2), 3); + EXPECT_EQ(values.get_element(3), 4); + EXPECT_EQ(values.get_element(4), 5); +} + +TEST(ArrayMapFunctionTest, NestedArraySortComparatorCapturingOuterArgumentReturnsError) { + auto int_type = std::make_shared(); + auto int8_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"i"})); + auto sort_call = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(array_int_type, 2, "array_sort")); + auto sort_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"x", "y"})); + auto add = std::make_shared(int_type); + + // This mirrors: + // array_map(i -> array_sort((x, y) -> x + i, arr2), arr1) + // array_sort's comparator executes against a two-column temporary block that + // only contains x and y. Capturing i from the outer array_map would otherwise + // be silently resolved by column id as x. + auto sort_x = VColumnRef::create_shared(make_column_ref_node(0, "x", int_type)); + auto outer_i = VColumnRef::create_shared(make_column_ref_node(0, "i", int_type)); + + add->add_child(sort_x); + add->add_child(outer_i); + sort_lambda->add_child(add); + sort_call->add_child(sort_lambda); + sort_call->add_child(std::make_shared(make_int_array_column({{2, 1}, {4, 3}}), + array_int_type, "arr2")); + outer_lambda->add_child(sort_call); + root->add_child(outer_lambda); + root->add_child(std::make_shared(make_int_array_column({{10, 20}, {30}}), + array_int_type, "arr1")); + + VExprContext context(root); + RuntimeState state; + RowDescriptor row_desc; + auto status = root->prepare(&state, row_desc, &context); + EXPECT_FALSE(status.ok()); + EXPECT_NE( + status.to_string().find("array_sort comparator only supports its own lambda arguments"), + std::string::npos); + EXPECT_NE(status.to_string().find("captured column ref 'i'"), std::string::npos); +} + +TEST(ArrayMapFunctionTest, + NestedArraySortComparatorNestedLambdaCapturingComparatorArgumentReturnsError) { + auto int_type = std::make_shared(); + auto int8_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + + auto root = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(array_int_type, 2, "array_sort")); + auto sort_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"x", "y"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"z"})); + auto add = std::make_shared(int_type); + + // This mirrors: + // array_sort((x, y) -> array_map(z -> z + x, [1, 2]), arr) + // The inner array_map's lambda frame cannot see array_sort comparator-local x/y because + // array_sort intentionally uses a position-based anonymous comparator frame. + auto inner_z = VColumnRef::create_shared(make_column_ref_node(0, "z", int_type)); + auto comparator_x = VColumnRef::create_shared(make_column_ref_node(0, "x", int_type)); + + add->add_child(inner_z); + add->add_child(comparator_x); + inner_lambda->add_child(add); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + sort_lambda->add_child(inner_call); + root->add_child(sort_lambda); + root->add_child(std::make_shared(make_int_array_column({{2, 1}, {4, 3}}), + array_int_type, "arr")); + + VExprContext context(root); + RuntimeState state; + RowDescriptor row_desc; + auto status = root->prepare(&state, row_desc, &context); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find( + "array_sort comparator does not support nested lambda capturing comparator " + "argument 'x'"), + std::string::npos); +} + +TEST(ArrayMapFunctionTest, NestedArraySortComparatorNestedLambdaCanShadowComparatorArgument) { + auto int_type = std::make_shared(); + auto int8_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + + auto root = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(array_int_type, 2, "array_sort")); + auto sort_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"x", "y"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"x"})); + + // This mirrors: + // array_sort((x, y) -> array_map(x -> x, [1, 2]), arr) + // The inner array_map argument named x shadows the array_sort comparator argument x. + auto inner_x = VColumnRef::create_shared(make_column_ref_node(0, "x", int_type)); + + inner_lambda->add_child(inner_x); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + sort_lambda->add_child(inner_call); + root->add_child(sort_lambda); + root->add_child(std::make_shared(make_int_array_column({{2, 1}, {4, 3}}), + array_int_type, "arr")); + + VExprContext context(root); + RuntimeState state; + RowDescriptor row_desc; + auto status = root->prepare(&state, row_desc, &context); + EXPECT_TRUE(status.ok()) << status.to_string(); +} + +TEST(ArrayMapFunctionTest, + NestedArraySortComparatorNestedLambdaCapturingNonComparatorColumnReturnsError) { + auto int_type = std::make_shared(); + auto int8_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + + auto root = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(array_int_type, 2, "array_sort")); + auto sort_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"x", "y"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"z"})); + auto add = std::make_shared(int_type); + + // This mirrors: + // array_sort((x, y) -> array_map(z -> z + outer_col, [1, 2]), arr) + // Only the nested lambda's own arguments are visible in nested lambda bodies. + auto inner_z = VColumnRef::create_shared(make_column_ref_node(0, "z", int_type)); + auto outer_col = VColumnRef::create_shared(make_column_ref_node(2, "outer_col", int_type)); + + add->add_child(inner_z); + add->add_child(outer_col); + inner_lambda->add_child(add); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + sort_lambda->add_child(inner_call); + root->add_child(sort_lambda); + root->add_child(std::make_shared(make_int_array_column({{2, 1}, {4, 3}}), + array_int_type, "arr")); + + VExprContext context(root); + RuntimeState state; + RowDescriptor row_desc; + auto status = root->prepare(&state, row_desc, &context); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find( + "array_sort comparator only supports nested lambda arguments inside nested " + "lambda bodies, but found captured column ref 'outer_col'"), + std::string::npos); +} + +TEST(ArrayMapFunctionTest, NestedArraySortComparatorNestedLambdaCapturingSlotRefReturnsError) { + auto int_type = std::make_shared(); + auto int8_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + + auto root = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(array_int_type, 2, "array_sort")); + auto sort_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"x", "y"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"z"})); + auto add = std::make_shared(int_type); + + // This mirrors: + // array_sort((x, y) -> array_map(z -> z + k1, [1, 2]), arr) + // SlotRefs from outside the nested lambda are not available inside array_sort's comparator. + auto inner_z = VColumnRef::create_shared(make_column_ref_node(0, "z", int_type)); + auto k1 = make_slot_ref(0, "k1", int_type); + + add->add_child(inner_z); + add->add_child(k1); + inner_lambda->add_child(add); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + sort_lambda->add_child(inner_call); + root->add_child(sort_lambda); + root->add_child(std::make_shared(make_int_array_column({{2, 1}, {4, 3}}), + array_int_type, "arr")); + + VExprContext context(root); + RuntimeState state; + RowDescriptor row_desc; + auto status = root->prepare(&state, row_desc, &context); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find( + "array_sort comparator only supports nested lambda arguments inside nested " + "lambda bodies, but found captured slot ref 'k1'"), + std::string::npos); +} + +TEST(ArrayMapFunctionTest, + NestedArraySortComparatorNestedLambdaWithoutArgumentMetadataReturnsError) { + auto int_type = std::make_shared(); + auto int8_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + + auto root = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(array_int_type, 2, "array_sort")); + auto sort_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"x", "y"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared( + make_lambda_expr_node(int_type, {"z"}, false /*set_argument_names*/)); + + // This mirrors an old FE plan where the nested array_map lambda has no argument metadata. + auto inner_z = VColumnRef::create_shared(make_column_ref_node(0, "z", int_type)); + + inner_lambda->add_child(inner_z); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + sort_lambda->add_child(inner_call); + root->add_child(sort_lambda); + root->add_child(std::make_shared(make_int_array_column({{2, 1}, {4, 3}}), + array_int_type, "arr")); + + VExprContext context(root); + RuntimeState state; + RowDescriptor row_desc; + auto status = root->prepare(&state, row_desc, &context); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find( + "Cannot validate nested lambda capture in array_sort comparator without " + "lambda metadata"), + std::string::npos); +} + +TEST(ArrayMapFunctionTest, + NestedArraySortComparatorTwoLevelNestedLambdaCanCaptureOuterNestedArgument) { + auto int_type = std::make_shared(); + auto int8_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(array_int_type, 2, "array_sort")); + auto sort_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"x", "y"})); + auto outer_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"z"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"q"})); + auto add = std::make_shared(int_type); + + // This mirrors: + // array_sort((x, y) -> array_map(z -> array_map(q -> q + z, [1, 2]), [3, 4]), arr) + // q is local to the inner array_map, while z is declared by the enclosing nested array_map. + auto inner_q = VColumnRef::create_shared(make_column_ref_node(0, "q", int_type)); + auto outer_z = VColumnRef::create_shared(make_column_ref_node(0, "z", int_type)); + + add->add_child(inner_q); + add->add_child(outer_z); + inner_lambda->add_child(add); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + outer_lambda->add_child(inner_call); + outer_call->add_child(outer_lambda); + outer_call->add_child(std::make_shared(make_int_array_column({{3, 4}}), + array_int_type, "outer_array")); + sort_lambda->add_child(outer_call); + root->add_child(sort_lambda); + root->add_child(std::make_shared(make_int_array_column({{2, 1}, {4, 3}}), + array_int_type, "arr")); + + VExprContext context(root); + RuntimeState state; + RowDescriptor row_desc; + auto status = root->prepare(&state, row_desc, &context); + EXPECT_TRUE(status.ok()) << status.to_string(); +} + +TEST(ArrayMapFunctionTest, + NestedArraySortComparatorNestedLambdaCallWithoutArgumentMetadataReturnsError) { + auto int_type = std::make_shared(); + auto int8_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(array_int_type, 2, "array_sort")); + auto sort_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int8_type, {"x", "y"})); + auto outer_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"z"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared( + make_lambda_expr_node(int_type, {"q"}, false /*set_argument_names*/)); + auto add = std::make_shared(int_type); + + // This mirrors an old FE plan where a nested lambda call under another nested lambda has no + // argument metadata. + auto inner_q = VColumnRef::create_shared(make_column_ref_node(0, "q", int_type)); + auto outer_z = VColumnRef::create_shared(make_column_ref_node(0, "z", int_type)); + + add->add_child(inner_q); + add->add_child(outer_z); + inner_lambda->add_child(add); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + outer_lambda->add_child(inner_call); + outer_call->add_child(outer_lambda); + outer_call->add_child(std::make_shared(make_int_array_column({{3, 4}}), + array_int_type, "outer_array")); + sort_lambda->add_child(outer_call); + root->add_child(sort_lambda); + root->add_child(std::make_shared(make_int_array_column({{2, 1}, {4, 3}}), + array_int_type, "arr")); + + VExprContext context(root); + RuntimeState state; + RowDescriptor row_desc; + auto status = root->prepare(&state, row_desc, &context); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find( + "Cannot validate nested lambda capture in array_sort comparator without " + "lambda metadata"), + std::string::npos); +} + +TEST(ArrayMapFunctionTest, NestedLambdaCapturesOuterSlotRefFromInnerBody) { + auto int_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"x"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"y"})); + auto add = std::make_shared(int_type); + + // This mirrors: + // array_map(x -> array_map(y -> y + k1, [1, 2]), arr) + // The outer array_map body does not reference k1 directly. The BE still + // needs to carry the full input block through the outer lambda block and + // inherit it into the inner lambda block. + auto inner_y = VColumnRef::create_shared(make_column_ref_node(0, "y", int_type)); + auto k1 = make_slot_ref(0, "k1", int_type); + + add->add_child(inner_y); + add->add_child(k1); + inner_lambda->add_child(add); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + outer_lambda->add_child(inner_call); + root->add_child(outer_lambda); + root->add_child(std::make_shared(make_int_array_column({{10, 20}, {30}}), + array_int_type, "outer_array")); + + VExprContext context(root); + open_expr(root, &context); + + Block block; + block.insert({make_int_column({100, 200}), int_type, "k1"}); + + ColumnPtr result; + auto status = root->execute_column(&context, &block, nullptr, block.rows(), result); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& outer_array = assert_cast(*result); + ASSERT_EQ(outer_array.size(), 2); + ASSERT_EQ(outer_array.get_offsets()[0], 2); + ASSERT_EQ(outer_array.get_offsets()[1], 3); + + const auto* nullable_inner_arrays = + check_and_get_column(&outer_array.get_data()); + ASSERT_NE(nullable_inner_arrays, nullptr); + const auto& inner_arrays = + assert_cast(nullable_inner_arrays->get_nested_column()); + ASSERT_EQ(inner_arrays.size(), 3); + ASSERT_EQ(inner_arrays.get_offsets()[0], 2); + ASSERT_EQ(inner_arrays.get_offsets()[1], 4); + ASSERT_EQ(inner_arrays.get_offsets()[2], 6); + + const auto* nullable_values = check_and_get_column(&inner_arrays.get_data()); + ASSERT_NE(nullable_values, nullptr); + const auto& values = assert_cast(nullable_values->get_nested_column()); + ASSERT_EQ(values.size(), 6); + EXPECT_EQ(values.get_element(0), 101); + EXPECT_EQ(values.get_element(1), 102); + EXPECT_EQ(values.get_element(2), 101); + EXPECT_EQ(values.get_element(3), 102); + EXPECT_EQ(values.get_element(4), 201); + EXPECT_EQ(values.get_element(5), 202); +} + +TEST(ArrayMapFunctionTest, LegacySingleLambdaWithoutArgumentMetadataUsesColumnId) { + auto int_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + + auto root = VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto lambda = VLambdaFunctionExpr::create_shared( + make_lambda_expr_node(int_type, {"x"}, false /*set_argument_names*/)); + + // Simulate an old FE plan without lambda_argument_names. Single-layer + // lambda can still use the ColumnRef id to bind argument position 0. + auto x = VColumnRef::create_shared(make_column_ref_node(0, "legacy_x", int_type)); + lambda->add_child(x); + root->add_child(lambda); + root->add_child(std::make_shared(make_int_array_column({{10, 20}}), + array_int_type, "array")); + + VExprContext context(root); + open_expr(root, &context); + + Block block; + ColumnPtr result; + auto status = root->execute_column(&context, &block, nullptr, 1, result); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& result_array = assert_cast(*result); + ASSERT_EQ(result_array.size(), 1); + ASSERT_EQ(result_array.get_offsets()[0], 2); + const auto* nullable_values = check_and_get_column(&result_array.get_data()); + ASSERT_NE(nullable_values, nullptr); + const auto& values = assert_cast(nullable_values->get_nested_column()); + ASSERT_EQ(values.size(), 2); + EXPECT_EQ(values.get_element(0), 10); + EXPECT_EQ(values.get_element(1), 20); +} + +TEST(ArrayMapFunctionTest, LegacyNestedLambdaWithoutArgumentMetadataReturnsError) { + auto int_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto outer_lambda = VLambdaFunctionExpr::create_shared( + make_lambda_expr_node(array_int_type, {"x"}, false /*set_argument_names*/)); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared( + make_lambda_expr_node(int_type, {"y"}, false /*set_argument_names*/)); + auto subtract = std::make_shared(int_type); + + auto outer_x = VColumnRef::create_shared(make_column_ref_node(0, "x", int_type)); + auto inner_y = VColumnRef::create_shared(make_column_ref_node(0, "y", int_type)); + + subtract->add_child(outer_x); + subtract->add_child(inner_y); + inner_lambda->add_child(subtract); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + outer_lambda->add_child(inner_call); + root->add_child(outer_lambda); + root->add_child(std::make_shared(make_int_array_column({{10, 20}}), + array_int_type, "outer_array")); + + VExprContext context(root); + RuntimeState state; + RowDescriptor row_desc; + auto status = root->prepare(&state, row_desc, &context); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find( + "Cannot resolve nested lambda argument without lambda metadata"), + std::string::npos); +} + +TEST(ArrayMapFunctionTest, NestedLambdaUsesOuterArgumentsAndInputSlotRefArray) { + auto int_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 3)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"x", "y"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"z"})); + auto subtract = std::make_shared(int_type); + auto add = std::make_shared(int_type); + auto multiply = std::make_shared(int_type); + + // This mirrors: + // array_map((x, y) -> array_map(z -> (y - z) * (x + z), arr3), arr1, arr2) + // arr3 is a sparse ordinary input SlotRef. Each lambda block must keep required + // input positions before appending its own arguments, so the inner array_map can + // use arr3 while resolving x/y/z from the nested lambda context by name. + auto y_for_subtract = VColumnRef::create_shared(make_column_ref_node(10, "y", int_type)); + auto z_for_subtract = VColumnRef::create_shared(make_column_ref_node(0, "z", int_type)); + auto x_for_add = VColumnRef::create_shared(make_column_ref_node(9, "x", int_type)); + auto z_for_add = VColumnRef::create_shared(make_column_ref_node(0, "z", int_type)); + auto arr3 = make_slot_ref(4, "arr3", array_int_type); + + subtract->add_child(y_for_subtract); + subtract->add_child(z_for_subtract); + add->add_child(x_for_add); + add->add_child(z_for_add); + multiply->add_child(subtract); + multiply->add_child(add); + inner_lambda->add_child(multiply); + inner_call->add_child(inner_lambda); + inner_call->add_child(arr3); + outer_lambda->add_child(inner_call); + root->add_child(outer_lambda); + root->add_child(std::make_shared(make_int_array_column({{10, 20}, {30}}), + array_int_type, "arr1")); + root->add_child(std::make_shared(make_int_array_column({{100, 200}, {300}}), + array_int_type, "arr2")); + + VExprContext context(root); + open_expr(root, &context); + + Block block; + block.insert({make_int_column({0, 0}), int_type, "unused0"}); + block.insert({make_int_column({1, 1}), int_type, "unused1"}); + block.insert({make_int_column({2, 2}), int_type, "unused2"}); + block.insert({make_int_column({3, 3}), int_type, "unused3"}); + block.insert({make_int_array_column({{1, 2}, {3}}), array_int_type, "arr3"}); + + ColumnPtr result; + auto status = root->execute_column(&context, &block, nullptr, block.rows(), result); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& outer_array = assert_cast(*result); + ASSERT_EQ(outer_array.size(), 2); + ASSERT_EQ(outer_array.get_offsets()[0], 2); + ASSERT_EQ(outer_array.get_offsets()[1], 3); + + const auto* nullable_inner_arrays = + check_and_get_column(&outer_array.get_data()); + ASSERT_NE(nullable_inner_arrays, nullptr); + const auto& inner_arrays = + assert_cast(nullable_inner_arrays->get_nested_column()); + ASSERT_EQ(inner_arrays.size(), 3); + ASSERT_EQ(inner_arrays.get_offsets()[0], 2); + ASSERT_EQ(inner_arrays.get_offsets()[1], 4); + ASSERT_EQ(inner_arrays.get_offsets()[2], 5); + + const auto* nullable_values = check_and_get_column(&inner_arrays.get_data()); + ASSERT_NE(nullable_values, nullptr); + const auto& values = assert_cast(nullable_values->get_nested_column()); + ASSERT_EQ(values.size(), 5); + EXPECT_EQ(values.get_element(0), 1089); + EXPECT_EQ(values.get_element(1), 1176); + EXPECT_EQ(values.get_element(2), 4179); + EXPECT_EQ(values.get_element(3), 4356); + EXPECT_EQ(values.get_element(4), 9801); +} + +TEST(ArrayMapFunctionTest, ThreeLevelNestedLambdaCapturesAllOuterArguments) { + auto int_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + auto three_level_array_int_type = std::make_shared(nested_array_int_type); + + auto root = VLambdaFunctionCallExpr::create_shared( + make_lambda_call_node(three_level_array_int_type, 2)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(nested_array_int_type, {"x"})); + auto middle_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto middle_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"y"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"z"})); + auto add_zy = std::make_shared(int_type); + auto add_zyx = std::make_shared(int_type); + + // This mirrors: + // array_map(x -> array_map(y -> array_map(z -> z + y + x, [1, 2]), + // [10, 20]), + // [100]) + // It verifies LambdaExecutionContext works as a stack: the innermost z + // shadows nothing, y is resolved from the middle frame, and x is resolved + // from the outer frame. + auto z = VColumnRef::create_shared(make_column_ref_node(0, "z", int_type)); + auto y = VColumnRef::create_shared(make_column_ref_node(7, "y", int_type)); + auto x = VColumnRef::create_shared(make_column_ref_node(9, "x", int_type)); + + add_zy->add_child(z); + add_zy->add_child(y); + add_zyx->add_child(add_zy); + add_zyx->add_child(x); + inner_lambda->add_child(add_zyx); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + middle_lambda->add_child(inner_call); + middle_call->add_child(middle_lambda); + middle_call->add_child(std::make_shared(make_int_array_column({{10, 20}}), + array_int_type, "middle_array")); + outer_lambda->add_child(middle_call); + root->add_child(outer_lambda); + root->add_child(std::make_shared(make_int_array_column({{100}}), array_int_type, + "outer_array")); + + VExprContext context(root); + open_expr(root, &context); + + Block block; + ColumnPtr result; + auto status = root->execute_column(&context, &block, nullptr, 1, result); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& level3_arrays = assert_cast(*result); + ASSERT_EQ(level3_arrays.size(), 1); + ASSERT_EQ(level3_arrays.get_offsets()[0], 1); + + const auto* nullable_level2_arrays = + check_and_get_column(&level3_arrays.get_data()); + ASSERT_NE(nullable_level2_arrays, nullptr); + const auto& level2_arrays = + assert_cast(nullable_level2_arrays->get_nested_column()); + ASSERT_EQ(level2_arrays.size(), 1); + ASSERT_EQ(level2_arrays.get_offsets()[0], 2); + + const auto* nullable_level1_arrays = + check_and_get_column(&level2_arrays.get_data()); + ASSERT_NE(nullable_level1_arrays, nullptr); + const auto& level1_arrays = + assert_cast(nullable_level1_arrays->get_nested_column()); + ASSERT_EQ(level1_arrays.size(), 2); + ASSERT_EQ(level1_arrays.get_offsets()[0], 2); + ASSERT_EQ(level1_arrays.get_offsets()[1], 4); + + const auto* nullable_values = check_and_get_column(&level1_arrays.get_data()); + ASSERT_NE(nullable_values, nullptr); + const auto& values = assert_cast(nullable_values->get_nested_column()); + ASSERT_EQ(values.size(), 4); + EXPECT_EQ(values.get_element(0), 111); + EXPECT_EQ(values.get_element(1), 112); + EXPECT_EQ(values.get_element(2), 121); + EXPECT_EQ(values.get_element(3), 122); +} + +TEST(ArrayMapFunctionTest, NestedLambdaCapturesOuterArgumentWithSameElementType) { + auto int_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"x"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"y"})); + auto subtract = std::make_shared(int_type); + + // This mirrors array_map(x -> array_map(y -> y - x, [1, 2]), [10, 20]). + // The captured outer x and the inner y both use INT type, so only names + // plus lambda scope can disambiguate them. The non-ordinal id of outer x + // covers FE plans where lambda argument ColumnRef ids are not 0-based. + auto outer_x = VColumnRef::create_shared(make_column_ref_node(1, "x", int_type)); + auto inner_y = VColumnRef::create_shared(make_column_ref_node(0, "y", int_type)); + + subtract->add_child(inner_y); + subtract->add_child(outer_x); + inner_lambda->add_child(subtract); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + outer_lambda->add_child(inner_call); + root->add_child(outer_lambda); + root->add_child(std::make_shared(make_int_array_column({{10, 20}}), + array_int_type, "outer_array")); + + VExprContext context(root); + open_expr(root, &context); + + // No ordinary input column is needed by this expression. The outer lambda + // block only appends its own x argument, while the inner lambda resolves y + // from the nearest frame and x from the outer frame. + Block block; + + ColumnPtr result; + auto status = root->execute_column(&context, &block, nullptr, 1, result); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& outer_array = assert_cast(*result); + ASSERT_EQ(outer_array.size(), 1); + ASSERT_EQ(outer_array.get_offsets()[0], 2); + + const auto* nullable_inner_arrays = + check_and_get_column(&outer_array.get_data()); + ASSERT_NE(nullable_inner_arrays, nullptr); + const auto& inner_arrays = + assert_cast(nullable_inner_arrays->get_nested_column()); + ASSERT_EQ(inner_arrays.size(), 2); + ASSERT_EQ(inner_arrays.get_offsets()[0], 2); + ASSERT_EQ(inner_arrays.get_offsets()[1], 4); + + const auto* nullable_values = check_and_get_column(&inner_arrays.get_data()); + ASSERT_NE(nullable_values, nullptr); + const auto& values = assert_cast(nullable_values->get_nested_column()); + ASSERT_EQ(values.size(), 4); + EXPECT_EQ(values.get_element(0), -9); + EXPECT_EQ(values.get_element(1), -8); + EXPECT_EQ(values.get_element(2), -19); + EXPECT_EQ(values.get_element(3), -18); +} + +TEST(ArrayMapFunctionTest, NestedLambdaCapturesOuterArgumentBeforeInnerArgument) { + auto int_type = std::make_shared(); + auto array_int_type = std::make_shared(int_type); + auto nested_array_int_type = std::make_shared(array_int_type); + + auto root = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(nested_array_int_type, 2)); + auto outer_lambda = + VLambdaFunctionExpr::create_shared(make_lambda_expr_node(array_int_type, {"x"})); + auto inner_call = + VLambdaFunctionCallExpr::create_shared(make_lambda_call_node(array_int_type, 2)); + auto inner_lambda = VLambdaFunctionExpr::create_shared(make_lambda_expr_node(int_type, {"y"})); + auto subtract = std::make_shared(int_type); + + // This mirrors array_map(x -> array_map(y -> x - y, [1, 2]), [10, 20]). + // The outer x and inner y both use INT type. FE-provided lambda argument + // names must make x bind to the outer lambda even when x is visited before + // y in the inner lambda body. The non-ordinal id of outer x covers FE plans + // where lambda argument ColumnRef ids are not 0-based. + auto outer_x = VColumnRef::create_shared(make_column_ref_node(1, "x", int_type)); + auto inner_y = VColumnRef::create_shared(make_column_ref_node(0, "y", int_type)); + + subtract->add_child(outer_x); + subtract->add_child(inner_y); + inner_lambda->add_child(subtract); + inner_call->add_child(inner_lambda); + inner_call->add_child(std::make_shared(make_int_array_column({{1, 2}}), + array_int_type, "inner_array")); + outer_lambda->add_child(inner_call); + root->add_child(outer_lambda); + root->add_child(std::make_shared(make_int_array_column({{10, 20}}), + array_int_type, "outer_array")); + + VExprContext context(root); + open_expr(root, &context); + + Block block; + + ColumnPtr result; + auto status = root->execute_column(&context, &block, nullptr, 1, result); + ASSERT_TRUE(status.ok()) << status.to_string(); + + const auto& outer_array = assert_cast(*result); + ASSERT_EQ(outer_array.size(), 1); + ASSERT_EQ(outer_array.get_offsets()[0], 2); + + const auto* nullable_inner_arrays = + check_and_get_column(&outer_array.get_data()); + ASSERT_NE(nullable_inner_arrays, nullptr); + const auto& inner_arrays = + assert_cast(nullable_inner_arrays->get_nested_column()); + ASSERT_EQ(inner_arrays.size(), 2); + ASSERT_EQ(inner_arrays.get_offsets()[0], 2); + ASSERT_EQ(inner_arrays.get_offsets()[1], 4); + + const auto* nullable_values = check_and_get_column(&inner_arrays.get_data()); + ASSERT_NE(nullable_values, nullptr); + const auto& values = assert_cast(nullable_values->get_nested_column()); + ASSERT_EQ(values.size(), 4); + EXPECT_EQ(values.get_element(0), 9); + EXPECT_EQ(values.get_element(1), 8); + EXPECT_EQ(values.get_element(2), 19); + EXPECT_EQ(values.get_element(3), 18); +} + +} // namespace doris diff --git a/be/test/exprs/vcolumn_ref_test.cpp b/be/test/exprs/vcolumn_ref_test.cpp new file mode 100644 index 00000000000000..c8eaa6094986fe --- /dev/null +++ b/be/test/exprs/vcolumn_ref_test.cpp @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/vcolumn_ref.h" + +#include +#include +#include + +#include +#include +#include + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "exprs/vexpr_context.h" +#include "runtime/descriptors.h" +#include "runtime/runtime_state.h" + +namespace doris { + +static TTypeDesc make_int_type_desc() { + TTypeDesc type_desc; + TTypeNode type_node; + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::INT); + type_node.__set_type(TTypeNodeType::SCALAR); + type_node.__set_scalar_type(scalar_type); + type_desc.types.push_back(type_node); + return type_desc; +} + +static TExprNode make_column_ref_node(int column_id, const std::string& column_name) { + TExprNode node; + node.__set_node_type(TExprNodeType::COLUMN_REF); + node.__set_num_children(0); + node.__set_type(make_int_type_desc()); + node.__set_is_nullable(false); + + TColumnRef column_ref; + column_ref.__set_column_id(column_id); + column_ref.__set_column_name(column_name); + node.__set_column_ref(column_ref); + return node; +} + +static ColumnPtr make_int_column(const std::vector& values) { + auto column = ColumnInt32::create(); + for (auto value : values) { + column->insert_value(value); + } + return column; +} + +static Block make_int_block() { + auto int_type = std::make_shared(); + Block block; + block.insert({make_int_column({10, 11}), int_type, "c0"}); + block.insert({make_int_column({20, 21}), int_type, "c1"}); + block.insert({make_int_column({30, 31}), int_type, "c2"}); + return block; +} + +static void open_expr(const VExprSPtr& expr, VExprContext* context) { + RuntimeState state; + RowDescriptor row_desc; + ASSERT_TRUE(expr->prepare(&state, row_desc, context).ok()); + ASSERT_TRUE(expr->open(&state, context, FunctionContext::THREAD_LOCAL).ok()); +} + +static std::vector get_int_values(const ColumnPtr& column) { + const auto* int_column = assert_cast(column.get()); + std::vector values; + for (auto value : int_column->get_data()) { + values.push_back(value); + } + return values; +} + +TEST(VColumnRefTest, SetGapOverridesPreviousGap) { + auto ref = VColumnRef::create_shared(make_column_ref_node(0, "x")); + VExprContext context(ref); + open_expr(ref, &context); + + auto block = make_int_block(); + ColumnPtr result; + + EXPECT_FALSE(ref->has_gap()); + ref->set_gap(0); + EXPECT_TRUE(ref->has_gap()); + + ref->set_gap(1); + auto status = ref->execute_column(&context, &block, nullptr, block.rows(), result); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(get_int_values(result), std::vector({20, 21})); + + result.reset(); + ref->set_gap(2); + status = ref->execute_column(&context, &block, nullptr, block.rows(), result); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(get_int_values(result), std::vector({30, 31})); +} + +TEST(VColumnRefTest, OutOfRangeColumnPositionReturnsError) { + auto ref = VColumnRef::create_shared(make_column_ref_node(1, "x")); + VExprContext context(ref); + open_expr(ref, &context); + + auto block = make_int_block(); + ref->set_gap(3); + + ColumnPtr result; + auto status = ref->execute_column(&context, &block, nullptr, block.rows(), result); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("input block not contain column ref x"), std::string::npos); + EXPECT_NE(status.to_string().find("column_id=1"), std::string::npos); + EXPECT_NE(status.to_string().find("gap=3"), std::string::npos); +} + +TEST(VColumnRefTest, OutOfRangeExecuteTypeThrowsException) { + auto ref = VColumnRef::create_shared(make_column_ref_node(1, "x")); + VExprContext context(ref); + open_expr(ref, &context); + + auto block = make_int_block(); + ref->set_gap(3); + + EXPECT_THROW({ static_cast(ref->execute_type(&block)); }, Exception); +} + +} // namespace doris diff --git a/be/test/exprs/vexpr_test.cpp b/be/test/exprs/vexpr_test.cpp index dc430d55915d6d..ddcc5234e0f2e6 100644 --- a/be/test/exprs/vexpr_test.cpp +++ b/be/test/exprs/vexpr_test.cpp @@ -797,8 +797,8 @@ TEST(TEST_VEXPR, LITERALTEST) { auto src_col = ColumnDecimal128V3::create(38, 6); auto& src_data = src_col->get_data(); src_data.resize(0); - src_data.push_back(Decimal128V3(123456789012345)); - src_data.push_back(Decimal128V3(-123456789012345)); + src_data.push_back(Decimal128V3(int64_t(123456789012345))); + src_data.push_back(Decimal128V3(int64_t(-123456789012345))); { auto node = std::make_shared( create_texpr_node_from(src_col->operator[](0), TYPE_DECIMAL128I, 38, 6), true); diff --git a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp index 7c517d49f61498..841e01f9cc91f5 100644 --- a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp +++ b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -36,12 +37,14 @@ #include "cloud/config.h" #include "common/config.h" #include "exec/common/endian.h" +#include "format/table/deletion_vector_reader.h" #include "io/fs/file_meta_cache.h" #include "roaring/roaring64map.hh" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" #include "testutil/mock/mock_runtime_state.h" +#include "util/hash_util.hpp" namespace doris { @@ -112,8 +115,8 @@ TIcebergDeleteFileDesc make_iceberg_deletion_vector(const std::string& path, int return delete_file; } -int64_t write_iceberg_deletion_vector_file(const std::string& file_path, - const std::vector& deleted_positions) { +std::vector build_iceberg_deletion_vector_blob( + const std::vector& deleted_positions) { roaring::Roaring64Map rows; for (const auto position : deleted_positions) { rows.add(position); @@ -127,7 +130,14 @@ int64_t write_iceberg_deletion_vector_file(const std::string& file_path, BigEndian::Store32(blob.data(), total_length); constexpr char DV_MAGIC[] = {'\xD1', '\xD3', '\x39', '\x64'}; memcpy(blob.data() + 4, DV_MAGIC, 4); - BigEndian::Store32(blob.data() + 8 + bitmap_size, 0); + const uint32_t crc = HashUtil::zlib_crc_hash(blob.data() + 4, total_length, 0); + BigEndian::Store32(blob.data() + 8 + bitmap_size, crc); + return blob; +} + +int64_t write_iceberg_deletion_vector_file(const std::string& file_path, + const std::vector& deleted_positions) { + const auto blob = build_iceberg_deletion_vector_blob(deleted_positions); std::ofstream output(file_path, std::ios::binary); EXPECT_TRUE(output.is_open()); @@ -271,6 +281,42 @@ TEST(IcebergDeleteFileReaderHelperTest, DeletionVectorCacheKeyEscapesPathBoundar build_iceberg_deletion_vector_cache_key(second_data_file_path, second_delete_file)); } +TEST(IcebergDeleteFileReaderHelperTest, ValidateDeletionVectorDescriptor) { + size_t bytes_read = 0; + + TIcebergDeleteFileDesc missing_path; + missing_path.__set_content_offset(0); + missing_path.__set_content_size_in_bytes(12); + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor(missing_path, bytes_read).ok()); + + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", -1, 12), bytes_read) + .ok()); + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 0, 11), bytes_read) + .ok()); + EXPECT_TRUE( + validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 0, MAX_ICEBERG_DELETION_VECTOR_BYTES), + bytes_read) + .ok()); + EXPECT_EQ(static_cast(MAX_ICEBERG_DELETION_VECTOR_BYTES), bytes_read); + const auto unsupported_status = validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 0, MAX_ICEBERG_DELETION_VECTOR_BYTES + 1), + bytes_read); + EXPECT_TRUE(unsupported_status.is()) << unsupported_status; + EXPECT_FALSE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", + std::numeric_limits::max() - 10, 12), + bytes_read) + .ok()); + + EXPECT_TRUE(validate_iceberg_deletion_vector_descriptor( + make_iceberg_deletion_vector("dv.puffin", 7, 12), bytes_read) + .ok()); + EXPECT_EQ(bytes_read, 12); +} + TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorReportsMissingFile) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_missing_test"; @@ -296,9 +342,9 @@ TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorReportsMissingFile) { std::filesystem::remove_all(test_dir); } -TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorReportsShortRead) { +TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorRejectsRangePastFile) { const auto test_dir = std::filesystem::temp_directory_path() / - "doris_iceberg_deletion_vector_short_read_test"; + "doris_iceberg_deletion_vector_range_past_file_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); @@ -316,11 +362,49 @@ TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorReportsShortRead) { auto status = read_iceberg_deletion_vector( make_iceberg_deletion_vector(dv_path, 0, dv_size + 1), options, &rows_to_delete); - EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("range exceeds file size"), std::string::npos); + EXPECT_NE(status.to_string().find(dv_path), std::string::npos); EXPECT_EQ(rows_to_delete.cardinality(), 0); std::filesystem::remove_all(test_dir); } +TEST(IcebergDeleteFileReaderHelperTest, DeletionVectorReaderValidatesOpenedFileRange) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_deletion_vector_opened_file_range_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto dv_path = (test_dir / "delete-vector.bin").string(); + const auto dv_size = write_iceberg_deletion_vector_file(dv_path, {1, 3}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + IcebergDeleteFileIOContext io_context(&state); + + { + TFileRangeDesc exact_range = build_iceberg_delete_file_range(dv_path); + exact_range.start_offset = 4; + exact_range.size = dv_size - exact_range.start_offset; + DeletionVectorReader exact_reader(&state, &profile, scan_params, exact_range, + &io_context.io_ctx); + const auto exact_status = exact_reader.open(); + EXPECT_TRUE(exact_status.ok()) << exact_status; + + TFileRangeDesc oversized_range = exact_range; + oversized_range.size = MAX_ICEBERG_DELETION_VECTOR_BYTES; + DeletionVectorReader oversized_reader(&state, &profile, scan_params, oversized_range, + &io_context.io_ctx); + const auto oversized_status = oversized_reader.open(); + EXPECT_TRUE(oversized_status.is()) << oversized_status; + EXPECT_NE(oversized_status.to_string().find("range exceeds file size"), std::string::npos); + EXPECT_NE(oversized_status.to_string().find(dv_path), std::string::npos); + } + + std::filesystem::remove_all(test_dir); +} + TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorStopsWhenIoContextStops) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_stop_test"; @@ -362,6 +446,19 @@ TEST(IcebergDeleteFileReaderHelperTest, DecodeDeletionVectorRejectsCorruptPayloa EXPECT_EQ(rows_to_delete.cardinality(), 0); } +TEST(IcebergDeleteFileReaderHelperTest, DecodeDeletionVectorRejectsCrcMismatch) { + auto corrupted = build_iceberg_deletion_vector_blob({1, 3}); + corrupted.back() ^= 1; + + roaring::Roaring64Map rows_to_delete; + auto status = decode_iceberg_deletion_vector_buffer(corrupted.data(), corrupted.size(), + &rows_to_delete); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("CRC32 mismatch"), std::string::npos); + EXPECT_EQ(rows_to_delete.cardinality(), 0); +} + TEST(IcebergDeleteFileReaderHelperTest, ReadDeletionVectorReadsMillionDeletePositions) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_deletion_vector_large_test"; diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp index bc3f916281de7d..e22060e4417043 100644 --- a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -17,9 +17,15 @@ #include "format/table/iceberg_position_delete_sys_table_reader.h" +#include +#include #include +#include +#include +#include #include +#include #include #include #include @@ -29,10 +35,14 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_struct.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "exprs/vexpr.h" +#include "format/orc/orc_memory_stream_test.h" +#include "format/table/iceberg_scan_semantics.h" #include "format/table/parquet_utils.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" #include "io/io_common.h" @@ -43,6 +53,8 @@ namespace doris { namespace { +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file); + class ProfileTrackingReader final : public GenericReader { public: int collect_calls = 0; @@ -55,6 +67,31 @@ class ProfileTrackingReader final : public GenericReader { void _collect_profile_before_close() override { ++collect_calls; } }; +class RejectAllRowsPredicate final : public VExpr { +public: + RejectAllRowsPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 0); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(); + return Status::OK(); + } + +private: + const std::string _name = "RejectAllRowsPredicate"; +}; + SlotDescriptor* make_slot(ObjectPool* pool, int id, std::string name, DataTypePtr type) { TSlotDescriptor slot_desc; slot_desc.__set_id(id); @@ -115,6 +152,386 @@ Int64 struct_int_at(const Block& block, const std::string& name, size_t child_in return child.get_nested_column().get_int(row); } +bool struct_child_is_null_at(const Block& block, const std::string& name, size_t child_index, + size_t row) { + const auto& struct_column = assert_cast(nested_column(block, name)); + const auto& child = assert_cast(struct_column.get_column(child_index)); + return child.is_null_at(row); +} + +std::shared_ptr int32_array(int32_t value) { + arrow::Int32Builder builder; + EXPECT_TRUE(builder.Append(value).ok()); + auto result = builder.Finish(); + EXPECT_TRUE(result.ok()) << result.status(); + return result.ok() ? *result : nullptr; +} + +void write_mixed_id_position_delete_parquet(const std::string& path, + std::optional row_id = 2147483544, + bool include_child_id = true) { + auto id = [](int32_t value) { + return arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(value)}); + }; + auto legacy_a = arrow::field("legacy_a", arrow::int32(), false); + if (include_child_id) { + legacy_a = legacy_a->WithMetadata(id(1)); + } + auto idless_b = arrow::field("b", arrow::int32(), false); + auto row_array = + arrow::StructArray::Make({int32_array(42), int32_array(99)}, {legacy_a, idless_b}); + ASSERT_TRUE(row_array.ok()) << row_array.status(); + + arrow::StringBuilder path_builder; + ASSERT_TRUE(path_builder.Append("s3://bucket/data.parquet").ok()); + auto path_array = path_builder.Finish(); + ASSERT_TRUE(path_array.ok()) << path_array.status(); + arrow::Int64Builder pos_builder; + ASSERT_TRUE(pos_builder.Append(5).ok()); + auto pos_array = pos_builder.Finish(); + ASSERT_TRUE(pos_array.ok()) << pos_array.status(); + + auto row_field = arrow::field("row", arrow::struct_({legacy_a, idless_b}), false); + if (row_id.has_value()) { + row_field = row_field->WithMetadata(id(*row_id)); + } + auto schema = arrow::schema({ + arrow::field("file_path", arrow::utf8(), false)->WithMetadata(id(2147483546)), + arrow::field("pos", arrow::int64(), false)->WithMetadata(id(2147483545)), + row_field, + }); + auto table = arrow::Table::Make(schema, {*path_array, *pos_array, *row_array}); + auto output = arrow::io::FileOutputStream::Open(path); + ASSERT_TRUE(output.ok()) << output.status(); + ::parquet::WriterProperties::Builder properties; + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +void write_nested_wrapper_position_delete_parquet(const std::string& path) { + auto id = [](int32_t value) { + return arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(value)}); + }; + auto legacy_a = arrow::field("legacy_a", arrow::int32(), false)->WithMetadata(id(1)); + auto s_array = arrow::StructArray::Make({int32_array(42)}, {legacy_a}); + ASSERT_TRUE(s_array.ok()) << s_array.status(); + auto idless_s = arrow::field("s", arrow::struct_({legacy_a}), false); + auto row_array = arrow::StructArray::Make({*s_array}, {idless_s}); + ASSERT_TRUE(row_array.ok()) << row_array.status(); + + arrow::StringBuilder path_builder; + ASSERT_TRUE(path_builder.Append("s3://bucket/data.parquet").ok()); + auto path_array = path_builder.Finish(); + ASSERT_TRUE(path_array.ok()) << path_array.status(); + arrow::Int64Builder pos_builder; + ASSERT_TRUE(pos_builder.Append(5).ok()); + auto pos_array = pos_builder.Finish(); + ASSERT_TRUE(pos_array.ok()) << pos_array.status(); + + auto schema = arrow::schema({ + arrow::field("file_path", arrow::utf8(), false)->WithMetadata(id(2147483546)), + arrow::field("pos", arrow::int64(), false)->WithMetadata(id(2147483545)), + arrow::field("row", arrow::struct_({idless_s}), false)->WithMetadata(id(2147483544)), + }); + auto table = arrow::Table::Make(schema, {*path_array, *pos_array, *row_array}); + auto output = arrow::io::FileOutputStream::Open(path); + ASSERT_TRUE(output.ok()) << output.status(); + ::parquet::WriterProperties::Builder properties; + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +void write_mixed_id_position_delete_orc(const std::string& path, + std::optional row_id = 2147483544, + bool include_child_id = true) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct>")); + type->getSubtype(0)->setAttribute("iceberg.id", "2147483546"); + type->getSubtype(1)->setAttribute("iceberg.id", "2147483545"); + if (row_id.has_value()) { + type->getSubtype(2)->setAttribute("iceberg.id", std::to_string(*row_id)); + } + if (include_child_id) { + type->getSubtype(2)->getSubtype(0)->setAttribute("iceberg.id", "1"); + } + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(1); + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& file_path = dynamic_cast<::orc::StringVectorBatch&>(*root.fields[0]); + auto& pos = dynamic_cast<::orc::LongVectorBatch&>(*root.fields[1]); + auto& row = dynamic_cast<::orc::StructVectorBatch&>(*root.fields[2]); + auto& legacy_a = dynamic_cast<::orc::LongVectorBatch&>(*row.fields[0]); + auto& idless_b = dynamic_cast<::orc::LongVectorBatch&>(*row.fields[1]); + std::string data_path = "s3://bucket/data.orc"; + file_path.data[0] = data_path.data(); + file_path.length[0] = data_path.size(); + pos.data[0] = 5; + legacy_a.data[0] = 42; + idless_b.data[0] = 99; + root.numElements = file_path.numElements = pos.numElements = row.numElements = + legacy_a.numElements = idless_b.numElements = 1; + writer->add(*batch); + writer->close(); + + std::ofstream output(path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +format::ColumnDefinition table_column(int32_t id, std::string name, DataTypePtr type) { + format::ColumnDefinition column; + column.identifier = Field::create_field(id); + column.name = std::move(name); + column.type = std::move(type); + return column; +} + +schema::external::TFieldPtr external_scalar_field(const std::string& name, int32_t id, + TPrimitiveType::type type) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + TColumnType column_type; + column_type.__set_type(type); + field->__set_type(column_type); + schema::external::TFieldPtr ptr; + ptr.__set_field_ptr(std::move(field)); + return ptr; +} + +schema::external::TStructField position_delete_table_schema() { + schema::external::TStructField row_children; + row_children.__set_fields({external_scalar_field("a", 1, TPrimitiveType::INT), + external_scalar_field("b", 2, TPrimitiveType::INT)}); + auto row = std::make_shared(); + row->__set_name("row"); + row->__set_id(2147483544); + TColumnType row_type; + row_type.__set_type(TPrimitiveType::STRUCT); + row->__set_type(row_type); + row->nestedField.__set_struct_field(std::move(row_children)); + row->__isset.nestedField = true; + schema::external::TFieldPtr row_ptr; + row_ptr.__set_field_ptr(std::move(row)); + + schema::external::TStructField root; + root.__set_fields({external_scalar_field("file_path", 2147483546, TPrimitiveType::STRING), + external_scalar_field("pos", 2147483545, TPrimitiveType::BIGINT), + std::move(row_ptr)}); + return root; +} + +void run_v1_idless_row_position_delete_test(TFileFormatType::type file_format, + const std::string& extension) { + const auto test_dir = std::filesystem::temp_directory_path() / + ("doris_v1_position_delete_idless_row_" + extension); + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto path = (test_dir / ("delete." + extension)).string(); + if (file_format == TFileFormatType::FORMAT_PARQUET) { + write_mixed_id_position_delete_parquet(path, std::nullopt, false); + } else { + write_mixed_id_position_delete_orc(path, std::nullopt, false); + } + + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto row_type = make_nullable(std::make_shared( + DataTypes {nullable_int32, nullable_int32}, Strings {"a", "b"})); + ObjectPool pool; + std::vector slots {make_slot(&pool, 0, "row", row_type)}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("UTC"); + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(file_format); + scan_params.__set_current_schema_id(-1); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(position_delete_table_schema()); + scan_params.__set_history_schema_info({std::move(current_schema)}); + auto io_ctx = std::make_shared(); + io::FileReaderStats file_reader_stats; + io_ctx->file_reader_stats = &file_reader_stats; + + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(file_format); + auto range = range_with_delete_file(delete_file); + range.__set_path(path); + range.__set_start_offset(0); + range.__set_size(static_cast(std::filesystem::file_size(path))); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + + IcebergPositionDeleteSysTableReader reader(slots, &state, &profile, range, &scan_params, io_ctx, + nullptr); + ReaderInitContext context; + ASSERT_TRUE(reader.init_reader(&context).ok()); + Block block = make_output_block(slots); + size_t read_rows = 0; + bool eof = false; + ASSERT_TRUE(reader.get_next_block(&block, &read_rows, &eof).ok()); + ASSERT_EQ(1, read_rows); + ASSERT_EQ(1, block.rows()); + // Top-level delete fields carry IDs, so an ID-less physical row must not bind by name. + EXPECT_TRUE(is_null_at(block, "row", 0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +void run_mixed_id_position_delete_test(format::FileFormat file_format, + TFileFormatType::type thrift_format, + const std::string& extension) { + const auto test_dir = std::filesystem::temp_directory_path() / + ("doris_iceberg_position_delete_mixed_id_" + extension); + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto path = (test_dir / ("delete." + extension)).string(); + if (file_format == format::FileFormat::PARQUET) { + write_mixed_id_position_delete_parquet(path); + } else { + write_mixed_id_position_delete_orc(path); + } + + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto row_type = make_nullable(std::make_shared( + DataTypes {nullable_int32, nullable_int32}, Strings {"a", "b"})); + auto a = table_column(1, "a", nullable_int32); + a.name_mapping = {"legacy_a"}; + a.has_name_mapping = true; + auto b = table_column(2, "b", nullable_int32); + auto row = table_column(2147483544, "row", row_type); + row.children = {a, b}; + std::vector projected_columns {row}; + + ObjectPool pool; + std::vector slots {make_slot(&pool, 0, "row", row_type)}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(thrift_format); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = file_format, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = &slots, + }) + .ok()); + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(thrift_format); + auto range = range_with_delete_file(delete_file); + range.__set_path(path); + range.__set_start_offset(0); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + format::SplitReadOptions split_options; + split_options.current_range = std::move(range); + split_options.current_split_format = file_format; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = make_output_block(slots); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(1, block.rows()); + ASSERT_FALSE(struct_child_is_null_at(block, "row", 0, 0)); + EXPECT_EQ(42, struct_int_at(block, "row", 0, 0)); + // Once any file field has an Iceberg id, the id-less sibling is absent rather than name-bound. + EXPECT_TRUE(struct_child_is_null_at(block, "row", 1, 0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +void run_v2_nested_wrapper_position_delete_test() { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v2_position_delete_nested_wrapper_parquet"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto path = (test_dir / "delete.parquet").string(); + write_nested_wrapper_position_delete_parquet(path); + + const auto nullable_int32 = make_nullable(std::make_shared()); + const auto s_type = make_nullable( + std::make_shared(DataTypes {nullable_int32}, Strings {"a"})); + const auto row_type = + make_nullable(std::make_shared(DataTypes {s_type}, Strings {"s"})); + auto a = table_column(1, "a", nullable_int32); + auto s = table_column(10, "s", s_type); + s.children = {a}; + auto row = table_column(2147483544, "row", row_type); + row.children = {s}; + std::vector projected_columns {row}; + + ObjectPool pool; + std::vector slots {make_slot(&pool, 0, "row", row_type)}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = format::FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = &slots, + }) + .ok()); + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_path(path); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + auto range = range_with_delete_file(delete_file); + range.__set_path(path); + range.__set_start_offset(0); + range.__set_file_size(static_cast(std::filesystem::file_size(path))); + format::SplitReadOptions split_options; + split_options.current_range = std::move(range); + split_options.current_split_format = format::FileFormat::PARQUET; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = make_output_block(slots); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(1, block.rows()); + EXPECT_EQ("{\"s\":{\"a\":42}}", row_type->to_string(*block.get_by_position(0).column, 0)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { TIcebergFileDesc iceberg_desc; iceberg_desc.__set_delete_files({delete_file}); @@ -502,6 +919,14 @@ TEST(IcebergPositionDeleteSysTableReaderTest, AppendsNullMetadataAndUsesDeletePa EXPECT_TRUE(eof); } +TEST(IcebergPositionDeleteSysTableReaderTest, ParquetUsesFullFileIdModeForIdlessRow) { + run_v1_idless_row_position_delete_test(TFileFormatType::FORMAT_PARQUET, "parquet"); +} + +TEST(IcebergPositionDeleteSysTableReaderTest, OrcUsesFullFileIdModeForIdlessRow) { + run_v1_idless_row_position_delete_test(TFileFormatType::FORMAT_ORC, "orc"); +} + TEST(IcebergPositionDeleteSysTableV2ReaderTest, RecordsDeletionVectorRows) { io::FileReaderStats file_reader_stats; auto scanner_io_ctx = std::make_shared(); @@ -603,4 +1028,63 @@ TEST(IcebergPositionDeleteSysTableV2ReaderTest, StopsBeforeExpandingDeletionVect EXPECT_TRUE(eof); } +TEST(IcebergPositionDeleteSysTableV2ReaderTest, + AllFilteredDeletionVectorYieldsBeforeObservingCancellation) { + ObjectPool pool; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + const auto nullable_int64 = make_nullable(std::make_shared()); + std::vector file_slot_descs { + make_slot(&pool, 0, "pos", nullable_int64), + }; + + auto conjunct = VExprContext::create_shared(std::make_shared()); + RowDescriptor row_desc; + ASSERT_TRUE(conjunct->prepare(&state, row_desc).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._io_ctx = std::make_shared(); + reader._file_slot_descs = &file_slot_descs; + reader._projected_columns.resize(file_slot_descs.size()); + reader._remaining_conjuncts = {conjunct}; + reader._has_split = true; + reader._delete_file_kind = + format::iceberg::IcebergPositionDeleteSysTableV2Reader::DeleteFileKind::DELETION_VECTOR; + reader._batch_size = 1; + reader._dv_positions.add(uint64_t {7}); + reader._dv_positions.add(uint64_t {9}); + reader._dv_positions.add(uint64_t {11}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block = make_output_block(file_slot_descs); + bool eof = false; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_FALSE(eof); + EXPECT_EQ(block.rows(), 0); + ASSERT_TRUE(reader._next_dv_position.has_value()); + EXPECT_EQ(**reader._next_dv_position, 9); + + reader._io_ctx->should_stop = true; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_TRUE(eof); + ASSERT_TRUE(reader._next_dv_position.has_value()); + EXPECT_EQ(**reader._next_dv_position, 9); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetRowUsesAnyFieldIdMapping) { + run_mixed_id_position_delete_test(format::FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET, + "parquet"); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, OrcRowUsesAnyFieldIdMapping) { + run_mixed_id_position_delete_test(format::FileFormat::ORC, TFileFormatType::FORMAT_ORC, "orc"); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetReadsNestedIdlessWrapper) { + run_v2_nested_wrapper_position_delete_test(); +} + } // namespace doris diff --git a/be/test/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp b/be/test/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp index e32153d1ef7f74..c15cb8931b7011 100644 --- a/be/test/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp @@ -28,6 +28,7 @@ #include #include +#include "common/consts.h" #include "common/object_pool.h" #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" @@ -1166,4 +1167,96 @@ TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_6) { } } -} // namespace doris \ No newline at end of file +// Regression coverage for _create_column_ids() driven by the schema-mapping StructNode: +// 1. Synthesized/metadata slots (e.g. the TopN global row-id column) are never serialized into +// the Iceberg schema tree, so they are absent from the node. Before the +// get_children().contains() guard, StructNode::children_column_exists() was called on such an +// unregistered name and hit DCHECK(children.contains(name)) -> abort in debug/ASAN builds +// (and threw std::out_of_range from .at() in release). The projected row-id slot reproduces +// that; with the guard it is skipped instead. +// 2. A projected column must resolve to its physical file column BY NAME through the node, not by +// Iceberg field id and not by its own (table) name, so partial-id / name-mapping files stay +// correct. Table column "a" maps to physical "legacy_a", while an unrelated "stale" column +// carries a field id that collides with the projected slot's default field id (0). The result +// must be "legacy_a"'s column id, never "stale"'s -- a regression to id-only or identity-name +// binding would produce a different set and fail here. +TEST_F(IcebergReaderCreateColumnIdsTest, parquet_name_mapped_and_synthesized_slots) { + // Physical Parquet schema (BY_NAME / partial-id file): + // legacy_a: id-less real data column -> column id 1 + // stale: unrelated column carrying stale id 0 -> column id 2 + FieldDescriptor field_desc; + FieldSchema legacy_a; + legacy_a.name = "legacy_a"; + legacy_a.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_BIGINT, true); + legacy_a.field_id = -1; + field_desc._fields.emplace_back(legacy_a); + FieldSchema stale; + stale.name = "stale"; + stale.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_BIGINT, true); + stale.field_id = 0; // collides with the projected slot's default col_unique_id (0) + field_desc._fields.emplace_back(stale); + + // Table column "a" resolves BY NAME to physical "legacy_a"; the synthesized global row-id + // column is intentionally NOT registered. + auto struct_node = std::make_shared(); + struct_node->add_children("a", "legacy_a", TableSchemaChangeHelper::ConstNode::get_instance()); + std::shared_ptr table_info_node = struct_node; + + SlotDescriptor a_slot; + a_slot._type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_BIGINT, true); + a_slot._col_name = "a"; + + SlotDescriptor row_id_slot; + row_id_slot._type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_BIGINT, true); + row_id_slot._col_name = BeConsts::GLOBAL_ROWID_COL; + + TupleDescriptor tuple_desc; + tuple_desc.add_slot(&a_slot); + tuple_desc.add_slot(&row_id_slot); + + // No abort on the synthesized slot; "a" resolves BY NAME to "legacy_a" (column id 1), never to + // "stale" (column id 2) through the colliding field id. + const ColumnIdResult result = + IcebergParquetReader::_create_column_ids(&field_desc, &tuple_desc, table_info_node); + EXPECT_EQ(result.column_ids, (std::set {1})); + EXPECT_TRUE(result.filter_column_ids.empty()); +} + +TEST_F(IcebergReaderCreateColumnIdsTest, orc_name_mapped_and_synthesized_slots) { + // Physical ORC schema (BY_NAME): legacy_a -> column id 1, stale -> column id 2. "stale" carries + // an iceberg.id attribute colliding with the projected slot's default field id (0). + std::unique_ptr orc_type( + orc::Type::buildTypeFromString("struct")); + orc_type->getSubtype(1)->setAttribute(IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, "0"); + + // Table column "a" resolves BY NAME to physical "legacy_a"; the synthesized global row-id + // column is intentionally NOT registered. + auto struct_node = std::make_shared(); + struct_node->add_children("a", "legacy_a", TableSchemaChangeHelper::ConstNode::get_instance()); + std::shared_ptr table_info_node = struct_node; + + SlotDescriptor a_slot; + a_slot._type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_BIGINT, true); + a_slot._col_name = "a"; + + SlotDescriptor row_id_slot; + row_id_slot._type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_BIGINT, true); + row_id_slot._col_name = BeConsts::GLOBAL_ROWID_COL; + + TupleDescriptor tuple_desc; + tuple_desc.add_slot(&a_slot); + tuple_desc.add_slot(&row_id_slot); + + // No abort on the synthesized slot; "a" resolves BY NAME to "legacy_a" (column id 1), never to + // "stale" (column id 2) through the colliding field id. + const ColumnIdResult result = + IcebergOrcReader::_create_column_ids(orc_type.get(), &tuple_desc, table_info_node); + EXPECT_EQ(result.column_ids, (std::set {1})); + EXPECT_TRUE(result.filter_column_ids.empty()); +} + +} // namespace doris diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 6c504e0b49a46b..297ea0848fb7be 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -60,6 +60,7 @@ #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_column_chunk_reader.h" #include "format/parquet/vparquet_reader.h" +#include "format/table/iceberg_scan_semantics.h" #include "io/fs/file_meta_cache.h" #include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" @@ -138,6 +139,46 @@ void write_iceberg_three_int_orc_file( output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); } +void write_iceberg_orc_containers_with_idless_struct_children(const std::string& file_path) { + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct>,attrs:map>>")); + type->getSubtype(0)->setAttribute("iceberg.id", "10"); + type->getSubtype(1)->setAttribute("iceberg.id", "20"); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(1); + auto& root = dynamic_cast<::orc::StructVectorBatch&>(*batch); + + auto& items = dynamic_cast<::orc::ListVectorBatch&>(*root.fields[0]); + auto& item = dynamic_cast<::orc::StructVectorBatch&>(*items.elements); + auto& item_a = dynamic_cast<::orc::LongVectorBatch&>(*item.fields[0]); + items.offsets[0] = 0; + items.offsets[1] = 1; + item_a.data[0] = 42; + items.numElements = item.numElements = item_a.numElements = 1; + + auto& attrs = dynamic_cast<::orc::MapVectorBatch&>(*root.fields[1]); + auto& attr_key = dynamic_cast<::orc::LongVectorBatch&>(*attrs.keys); + auto& attr_value = dynamic_cast<::orc::StructVectorBatch&>(*attrs.elements); + auto& attr_a = dynamic_cast<::orc::LongVectorBatch&>(*attr_value.fields[0]); + attrs.offsets[0] = 0; + attrs.offsets[1] = 1; + attr_key.data[0] = 1; + attr_a.data[0] = 43; + attrs.numElements = attr_key.numElements = attr_value.numElements = attr_a.numElements = 1; + + root.numElements = 1; + writer->add(*batch); + writer->close(); + + std::ofstream output(file_path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + std::shared_ptr build_iceberg_int32_array(const std::vector& values) { arrow::Int32Builder builder; for (const auto value : values) { @@ -183,6 +224,52 @@ void write_iceberg_three_int_parquet_file( properties.build())); } +void write_iceberg_id_and_idless_struct_parquet_file(const std::string& file_path, int32_t id, + int32_t nested_value) { + auto id_field = arrow::field("id", arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"1"})); + auto child_field = + arrow::field("a", arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"2"})); + auto struct_result = + arrow::StructArray::Make({build_iceberg_int32_array({nested_value})}, {child_field}); + DORIS_CHECK(struct_result.ok()); + auto struct_field = arrow::field("s", arrow::struct_({child_field}), false); + auto table = arrow::Table::Make(arrow::schema({id_field, struct_field}), + {build_iceberg_int32_array({id}), *struct_result}); + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +void write_iceberg_id_and_struct_with_idless_child_parquet_file(const std::string& file_path, + int32_t id, int32_t nested_value) { + auto id_field = arrow::field("id", arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"1"})); + auto child_field = arrow::field("legacy_a", arrow::int32(), false); + auto struct_result = + arrow::StructArray::Make({build_iceberg_int32_array({nested_value})}, {child_field}); + DORIS_CHECK(struct_result.ok()); + auto struct_field = + arrow::field("s", arrow::struct_({child_field}), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + auto table = arrow::Table::Make(arrow::schema({id_field, struct_field}), + {build_iceberg_int32_array({id}), *struct_result}); + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + void write_iceberg_int_equality_delete_parquet_file(const std::string& file_path, const std::string& field_name, int32_t field_id, int32_t value) { @@ -245,6 +332,128 @@ Status create_single_int_tuple_descriptor(ObjectPool* object_pool, const std::st return Status::OK(); } +Status create_single_struct_tuple_descriptor(ObjectPool* object_pool, + DescriptorTbl** descriptor_table, + const TupleDescriptor** tuple_descriptor) { + TDescriptorTable thrift_table; + TTableDescriptor table_descriptor; + table_descriptor.__set_id(0); + table_descriptor.__set_tableType(TTableType::OLAP_TABLE); + table_descriptor.__set_numCols(1); + table_descriptor.__set_numClusteringCols(0); + thrift_table.__set_tableDescriptors({table_descriptor}); + + TStructField child_field; + child_field.__set_name("a"); + child_field.__set_contains_null(true); + TTypeNode struct_node; + struct_node.__set_type(TTypeNodeType::STRUCT); + struct_node.__set_struct_fields({child_field}); + TTypeNode child_node; + child_node.__set_type(TTypeNodeType::SCALAR); + TScalarType child_scalar; + child_scalar.__set_type(TPrimitiveType::INT); + child_node.__set_scalar_type(child_scalar); + TTypeDesc type; + type.__set_types({struct_node, child_node}); + + TSlotDescriptor slot_descriptor; + slot_descriptor.__set_id(0); + slot_descriptor.__set_parent(0); + slot_descriptor.__set_col_unique_id(10); + slot_descriptor.__set_slotType(type); + slot_descriptor.__set_columnPos(0); + slot_descriptor.__set_byteOffset(0); + slot_descriptor.__set_nullIndicatorByte(0); + slot_descriptor.__set_nullIndicatorBit(-1); + slot_descriptor.__set_colName("s"); + slot_descriptor.__set_slotIdx(0); + slot_descriptor.__set_isMaterialized(true); + thrift_table.__set_slotDescriptors({slot_descriptor}); + + TTupleDescriptor thrift_tuple; + thrift_tuple.__set_id(0); + thrift_tuple.__set_byteSize(16); + thrift_tuple.__set_numNullBytes(0); + thrift_tuple.__set_tableId(0); + thrift_table.__set_tupleDescriptors({thrift_tuple}); + RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); + *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); + return Status::OK(); +} + +Status create_array_map_struct_tuple_descriptor(ObjectPool* object_pool, + DescriptorTbl** descriptor_table, + const TupleDescriptor** tuple_descriptor) { + TDescriptorTable thrift_table; + TTableDescriptor table_descriptor; + table_descriptor.__set_id(0); + table_descriptor.__set_tableType(TTableType::OLAP_TABLE); + table_descriptor.__set_numCols(2); + table_descriptor.__set_numClusteringCols(0); + thrift_table.__set_tableDescriptors({table_descriptor}); + + auto scalar_int_node = [] { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar; + scalar.__set_type(TPrimitiveType::INT); + node.__set_scalar_type(scalar); + return node; + }; + auto struct_a_node = [] { + TTypeNode node; + node.__set_type(TTypeNodeType::STRUCT); + TStructField child; + child.__set_name("a"); + child.__set_contains_null(true); + node.__set_struct_fields({child}); + return node; + }; + + TTypeNode array_node; + array_node.__set_type(TTypeNodeType::ARRAY); + array_node.__set_contains_nulls({true}); + TTypeDesc items_type; + items_type.__set_types({array_node, struct_a_node(), scalar_int_node()}); + + TTypeNode map_node; + map_node.__set_type(TTypeNodeType::MAP); + map_node.__set_contains_nulls({false, true}); + TTypeDesc attrs_type; + attrs_type.__set_types({map_node, scalar_int_node(), struct_a_node(), scalar_int_node()}); + + std::vector slots; + auto add_slot = [&slots](int id, const std::string& name, int field_id, const TTypeDesc& type) { + TSlotDescriptor slot; + slot.__set_id(id); + slot.__set_parent(0); + slot.__set_col_unique_id(field_id); + slot.__set_slotType(type); + slot.__set_columnPos(id); + slot.__set_byteOffset(0); + slot.__set_nullIndicatorByte(0); + slot.__set_nullIndicatorBit(-1); + slot.__set_colName(name); + slot.__set_slotIdx(id); + slot.__set_isMaterialized(true); + slots.emplace_back(std::move(slot)); + }; + add_slot(0, "items", 10, items_type); + add_slot(1, "attrs", 20, attrs_type); + thrift_table.__set_slotDescriptors(std::move(slots)); + + TTupleDescriptor thrift_tuple; + thrift_tuple.__set_id(0); + thrift_tuple.__set_byteSize(32); + thrift_tuple.__set_numNullBytes(0); + thrift_tuple.__set_tableId(0); + thrift_table.__set_tupleDescriptors({thrift_tuple}); + RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); + *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); + return Status::OK(); +} + schema::external::TFieldPtr make_external_int_field(const std::string& name, int32_t field_id, std::optional initial_default, std::vector aliases = {}) { @@ -1075,18 +1284,24 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul return field_ptr; }; + static_assert(sizeof("0123456789abcdef0123456789abcdef") - 1 > StringView::kInlineSize); + const std::string binary_default = "0123456789abcdef0123456789abcdef"; + schema::external::TStructField root_field; root_field.__set_fields( {make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, "2024-01-01 00:00:00.123456", -1, 6, false), - make_field("added_binary", 2, TPrimitiveType::VARBINARY, "AAEC/w==", 4, -1, true), - make_field("added_string_binary", 3, TPrimitiveType::STRING, "AAEC/w==", -1, -1, - true)}); + make_field("added_binary", 2, TPrimitiveType::VARBINARY, + "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + static_cast(binary_default.size()), -1, true), + make_field("added_string_binary", 3, TPrimitiveType::STRING, + "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", -1, -1, true)}); schema::external::TSchema current_schema; current_schema.__set_schema_id(-1); current_schema.__set_root_field(root_field); TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); scan_params.__set_file_type(TFileType::FILE_LOCAL); scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); scan_params.__set_current_schema_id(-1); @@ -1102,7 +1317,8 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul &runtime_state, cache.get()); const auto timestamp_type = make_nullable(std::make_shared(6)); - const auto varbinary_type = make_nullable(std::make_shared(4)); + const auto varbinary_type = make_nullable( + std::make_shared(static_cast(binary_default.size()))); const auto string_type = make_nullable(std::make_shared()); Block block; block.insert({timestamp_type->create_column(), timestamp_type, "added_timestamp"}); @@ -1125,15 +1341,108 @@ TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaul ASSERT_EQ(block.rows(), 3); EXPECT_EQ(timestamp_type->to_string(*block.get_by_position(0).column, 0), "2024-01-01 00:00:00.123456"); - EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, 0), - std::string("\x00\x01\x02\xff", 4)); - EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, 0), - std::string("\x00\x01\x02\xff", 4)); + EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, 0), binary_default); + EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, 0), binary_default); EXPECT_FALSE(is_column_const(*block.get_by_position(0).column)); EXPECT_FALSE(is_column_const(*block.get_by_position(1).column)); EXPECT_FALSE(is_column_const(*block.get_by_position(2).column)); } +TEST_F(IcebergReaderTest, v1_top_level_missing_binary_prefers_iceberg_initial_default) { + auto field = std::make_shared(); + field->__set_name("added_uuid"); + field->__set_id(1); + field->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + field->__set_initial_default_value_is_base64(true); + TColumnType thrift_type; + thrift_type.__set_type(TPrimitiveType::VARBINARY); + thrift_type.__set_len(16); + field->__set_type(thrift_type); + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + schema::external::TStructField root_field; + root_field.__set_fields({std::move(field_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(std::move(root_field)); + + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({std::move(current_schema)}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state = RuntimeState(TQueryOptions(), TQueryGlobals()); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + + const auto varbinary_type = make_nullable(std::make_shared(16)); + Block block; + block.insert({varbinary_type->create_column(), varbinary_type, "added_uuid"}); + std::unordered_map positions = {{"added_uuid", 0}}; + reader.TEST_set_column_name_to_block_index(&positions); + + ASSERT_TRUE(reader.on_fill_missing_columns(&block, 2, {"added_uuid"}).ok()); + + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(0).column, 0), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); +} + +TEST_F(IcebergReaderTest, v1_legacy_plan_keeps_missing_binary_null) { + auto field = std::make_shared(); + field->__set_name("added_uuid"); + field->__set_id(1); + field->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + field->__set_initial_default_value_is_base64(true); + TColumnType thrift_type; + thrift_type.__set_type(TPrimitiveType::VARBINARY); + thrift_type.__set_len(16); + field->__set_type(thrift_type); + schema::external::TFieldPtr field_ptr; + field_ptr.__set_field_ptr(std::move(field)); + schema::external::TStructField root_field; + root_field.__set_fields({std::move(field_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(std::move(root_field)); + + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_file_type(TFileType::FILE_LOCAL); + old_fe_scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + old_fe_scan_params.__set_current_schema_id(-1); + old_fe_scan_params.__set_history_schema_info({std::move(current_schema)}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, old_fe_scan_params, scan_range, 1024, &ctz, + &io_ctx, &runtime_state, cache.get()); + + const auto varbinary_type = make_nullable(std::make_shared(16)); + Block block; + block.insert({varbinary_type->create_column(), varbinary_type, "added_uuid"}); + std::unordered_map positions = {{"added_uuid", 0}}; + reader.TEST_set_column_name_to_block_index(&positions); + reader._fill_col_name_to_block_idx = &positions; + + ASSERT_TRUE(reader.on_fill_missing_columns(&block, 2, {"added_uuid"}).ok()); + const auto& nullable = assert_cast(*block.get_by_position(0).column); + ASSERT_EQ(nullable.size(), 2); + EXPECT_TRUE(nullable.is_null_at(0)); + EXPECT_TRUE(nullable.is_null_at(1)); +} + TEST_F(IcebergReaderTest, v1_multi_equality_delete_hashes_materialized_missing_default) { schema::external::TStructField root_field; root_field.__set_fields({make_external_int_field("id", 0, std::nullopt), @@ -1143,6 +1452,7 @@ TEST_F(IcebergReaderTest, v1_multi_equality_delete_hashes_materialized_missing_d current_schema.__set_root_field(root_field); TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); scan_params.__set_current_schema_id(-1); scan_params.__set_history_schema_info({current_schema}); TFileRangeDesc scan_range; @@ -1196,6 +1506,7 @@ TEST_F(IcebergReaderTest, v1_missing_equality_key_returns_error_for_pruned_descr current_schema.__set_root_field(root_field); TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); scan_params.__set_current_schema_id(-1); scan_params.__set_history_schema_info({current_schema}); TFileRangeDesc scan_range; @@ -1233,6 +1544,7 @@ TEST_F(IcebergReaderTest, v1_orc_equality_delete_matches_missing_initial_default current_schema.__set_root_field(root_field); TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); scan_params.__set_file_type(TFileType::FILE_LOCAL); scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); scan_params.__set_current_schema_id(-1); @@ -1299,29 +1611,339 @@ TEST_F(IcebergReaderTest, v1_orc_equality_delete_matches_missing_initial_default std::filesystem::remove_all(test_dir); } -TEST_F(IcebergReaderTest, v1_parquet_partial_id_equality_delete_ignores_stale_field_id) { +TEST_F(IcebergReaderTest, v1_parquet_reads_idless_wrapper_with_authoritative_empty_mapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_parquet_authoritative_empty_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.parquet").string(); + write_iceberg_id_and_idless_struct_parquet_file(data_file, 1, 42); + + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_children; + struct_children.__set_fields({child_ptr}); + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->__set_name_mapping({}); + struct_field->__set_name_mapping_is_authoritative(true); + struct_field->nestedField.__set_struct_field(struct_children); + struct_field->__isset.nestedField = true; + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 1, std::nullopt), std::move(struct_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_struct_tuple_descriptor(&object_pool, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone("UTC", ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + io::FileReaderSPtr file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(data_file, &file_reader).ok()); + reader.set_file_reader(file_reader); + + std::vector column_descriptors(1); + column_descriptors[0].name = "s"; + std::unordered_map block_positions = {{"s", 0}}; + ParquetInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto child_type = make_nullable(std::make_shared()); + const auto result_type = + make_nullable(std::make_shared(DataTypes {child_type}, Strings {"a"})); + Block block; + block.insert({result_type->create_column(), result_type, "s"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 1); + EXPECT_EQ(result_type->to_string(*block.get_by_position(0).column, 0), "{\"a\":42}"); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_parquet_keeps_file_id_mode_inside_nested_struct) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_parquet_whole_file_id_mode_nested_struct_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.parquet").string(); + write_iceberg_id_and_struct_with_idless_child_parquet_file(data_file, 1, 42); + + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + child_field->__set_name_mapping({"legacy_a"}); + child_field->__set_initial_default_value("7"); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_children; + struct_children.__set_fields({child_ptr}); + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->nestedField.__set_struct_field(struct_children); + struct_field->__isset.nestedField = true; + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 1, std::nullopt), std::move(struct_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_struct_tuple_descriptor(&object_pool, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone("UTC", ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + io::FileReaderSPtr file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(data_file, &file_reader).ok()); + reader.set_file_reader(file_reader); + + std::vector column_descriptors(1); + column_descriptors[0].name = "s"; + std::unordered_map block_positions = {{"s", 0}}; + ParquetInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto child_type = make_nullable(std::make_shared()); + const auto result_type = + make_nullable(std::make_shared(DataTypes {child_type}, Strings {"a"})); + Block block; + block.insert({result_type->create_column(), result_type, "s"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 1); + EXPECT_EQ(result_type->to_string(*block.get_by_position(0).column, 0), "{\"a\":7}"); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_orc_keeps_file_id_mode_inside_array_and_map) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_orc_whole_file_id_mode_containers_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.orc").string(); + write_iceberg_orc_containers_with_idless_struct_children(data_file); + + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + auto make_struct_field = [&struct_type](const std::string& name, int32_t id, int32_t child_id, + const std::string& default_value) { + schema::external::TStructField children; + children.__set_fields({make_external_int_field("a", child_id, default_value)}); + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_type(struct_type); + field->nestedField.__set_struct_field(std::move(children)); + field->__isset.nestedField = true; + schema::external::TFieldPtr ptr; + ptr.__set_field_ptr(std::move(field)); + return ptr; + }; + + TColumnType array_type; + array_type.__set_type(TPrimitiveType::ARRAY); + auto items = std::make_shared(); + items->__set_name("items"); + items->__set_id(10); + items->__set_type(array_type); + schema::external::TArrayField array_field; + array_field.__set_item_field(make_struct_field("element", 11, 12, "7")); + items->nestedField.__set_array_field(std::move(array_field)); + items->__isset.nestedField = true; + schema::external::TFieldPtr items_ptr; + items_ptr.__set_field_ptr(std::move(items)); + + TColumnType map_type; + map_type.__set_type(TPrimitiveType::MAP); + auto attrs = std::make_shared(); + attrs->__set_name("attrs"); + attrs->__set_id(20); + attrs->__set_type(map_type); + schema::external::TMapField map_field; + map_field.__set_key_field(make_external_int_field("key", 21, std::nullopt)); + map_field.__set_value_field(make_struct_field("value", 22, 23, "8")); + attrs->nestedField.__set_map_field(std::move(map_field)); + attrs->__isset.nestedField = true; + schema::external::TFieldPtr attrs_ptr; + attrs_ptr.__set_field_ptr(std::move(attrs)); + + schema::external::TStructField root_field; + root_field.__set_fields({std::move(items_ptr), std::move(attrs_ptr)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(std::move(root_field)); + + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_array_map_struct_tuple_descriptor(&object_pool, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergOrcReader reader(&kv_cache, &profile, &runtime_state, scan_params, scan_range, 1024, + "UTC", &io_ctx, cache.get()); + + std::vector column_descriptors(2); + column_descriptors[0].name = "items"; + column_descriptors[1].name = "attrs"; + std::unordered_map block_positions = {{"items", 0}, {"attrs", 1}}; + OrcInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + Block block; + for (const auto* slot : tuple_descriptor->slots()) { + const auto& type = slot->get_data_type_ptr(); + block.insert({type->create_column(), type, slot->col_name()}); + } + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 1); + EXPECT_EQ(tuple_descriptor->slots()[0]->get_data_type_ptr()->to_string( + *block.get_by_position(0).column, 0), + "[{\"a\":7}]"); + EXPECT_EQ(tuple_descriptor->slots()[1]->get_data_type_ptr()->to_string( + *block.get_by_position(1).column, 0), + "{1:{\"a\":8}}"); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_parquet_mixed_ids_prefer_existing_equality_field_id) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_v1_parquet_partial_id_equality_delete_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto data_file = (test_dir / "data.parquet").string(); const auto delete_file = (test_dir / "equality-delete.parquet").string(); - // The id-less columns put the complete Parquet file in BY_NAME mode. The unrelated - // stale_added column deliberately retains field id 1, which must not override the historical - // alias selected for the hidden added_column equality key. - write_iceberg_three_int_parquet_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", - std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); - write_iceberg_int_equality_delete_parquet_file(delete_file, "added_column", 1, 7); + // The hidden equality-delete column keeps the old physical name after the table field is + // renamed. An unrelated ID-less sibling must not downgrade its authoritative ID lookup. + write_iceberg_three_int_parquet_file(data_file, "id", 0, {1, 2, 3}, "unrelated", std::nullopt, + {70, 70, 70}, "old_key", 1, {5, 7, 9}); + write_iceberg_int_equality_delete_parquet_file(delete_file, "old_key", 1, 7); schema::external::TStructField root_field; - root_field.__set_fields( - {make_external_int_field("id", 0, std::nullopt), - make_external_int_field("added_column", 1, std::nullopt, {"legacy_added"})}); + root_field.__set_fields({make_external_int_field("id", 0, std::nullopt), + make_external_int_field("new_key", 1, std::nullopt, {"old_key"})}); schema::external::TSchema current_schema; current_schema.__set_schema_id(-1); current_schema.__set_root_field(root_field); TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); scan_params.__set_file_type(TFileType::FILE_LOCAL); scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); scan_params.__set_current_schema_id(-1); @@ -1392,32 +2014,59 @@ TEST_F(IcebergReaderTest, v1_parquet_partial_id_equality_delete_ignores_stale_fi EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 0), "1"); EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "3"); + TFileScanRangeParams legacy_scan_params = scan_params; + legacy_scan_params.__isset.iceberg_scan_semantics_version = false; + ShardedKVCache legacy_kv_cache(8); + IcebergParquetReader legacy_reader(&legacy_kv_cache, &profile, legacy_scan_params, scan_range, + 1024, &ctz, &io_ctx, &runtime_state, cache.get()); + io::FileReaderSPtr legacy_file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(data_file, &legacy_file_reader).ok()); + legacy_reader.set_file_reader(legacy_file_reader); + ParquetInitContext legacy_context; + legacy_context.column_descs = &column_descriptors; + legacy_context.col_name_to_block_idx = &block_positions; + legacy_context.tuple_descriptor = tuple_descriptor; + legacy_context.params = &legacy_scan_params; + legacy_context.range = &scan_range; + const auto legacy_init_status = legacy_reader.init_reader(&legacy_context); + ASSERT_TRUE(legacy_init_status.ok()) << legacy_init_status; + + Block legacy_block; + legacy_block.insert({id_type->create_column(), id_type, "id"}); + size_t legacy_read_rows = 0; + bool legacy_eof = false; + const auto legacy_status = + legacy_reader.get_next_block(&legacy_block, &legacy_read_rows, &legacy_eof); + ASSERT_TRUE(legacy_status.ok()) << legacy_status; + ASSERT_EQ(legacy_read_rows, 2); + ASSERT_EQ(legacy_block.rows(), 2); + EXPECT_EQ(id_type->to_string(*legacy_block.get_by_position(0).column, 0), "1"); + EXPECT_EQ(id_type->to_string(*legacy_block.get_by_position(0).column, 1), "3"); + std::filesystem::remove_all(test_dir); } -TEST_F(IcebergReaderTest, v1_orc_partial_id_equality_delete_ignores_stale_field_id) { +TEST_F(IcebergReaderTest, v1_orc_mixed_ids_prefer_existing_equality_field_id) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_v1_orc_partial_id_equality_delete_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto data_file = (test_dir / "data.orc").string(); const auto delete_file = (test_dir / "equality-delete.orc").string(); - // One id-less column switches the whole file to BY_NAME. The hidden current key is physically - // stored as its id-less historical alias, while an unrelated migrated column still carries - // the key's stale field id. Equality-delete binding must select legacy_added, not stale_added. - write_iceberg_three_int_orc_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", - std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); - write_iceberg_int_orc_file(delete_file, "added_column", 1, {7}); + // Exercise the same renamed, unprojected key with an old physical name in ORC. + write_iceberg_three_int_orc_file(data_file, "id", 0, {1, 2, 3}, "unrelated", std::nullopt, + {70, 70, 70}, "old_key", 1, {5, 7, 9}); + write_iceberg_int_orc_file(delete_file, "old_key", 1, {7}); schema::external::TStructField root_field; - root_field.__set_fields( - {make_external_int_field("id", 0, std::nullopt), - make_external_int_field("added_column", 1, std::nullopt, {"legacy_added"})}); + root_field.__set_fields({make_external_int_field("id", 0, std::nullopt), + make_external_int_field("new_key", 1, std::nullopt, {"old_key"})}); schema::external::TSchema current_schema; current_schema.__set_schema_id(-1); current_schema.__set_root_field(root_field); TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); scan_params.__set_file_type(TFileType::FILE_LOCAL); scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); scan_params.__set_current_schema_id(-1); diff --git a/be/test/format/table/paimon_cpp_reader_test.cpp b/be/test/format/table/paimon_cpp_reader_test.cpp index e323f4f1af44f6..f2602c34fd010e 100644 --- a/be/test/format/table/paimon_cpp_reader_test.cpp +++ b/be/test/format/table/paimon_cpp_reader_test.cpp @@ -21,12 +21,14 @@ #include #include +#include #include #include #include "core/block/block.h" #include "exec/common/endian.h" #include "format/format_common.h" +#include "format/table/deletion_vector_reader.h" #include "format/table/paimon_reader.h" #include "io/fs/file_meta_cache.h" #include "io/io_common.h" @@ -172,9 +174,19 @@ TEST(PaimonDeletionVectorTest, RejectShortBuffer) { decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); ASSERT_FALSE(status.ok()); + EXPECT_TRUE(status.is()); EXPECT_NE(status.to_string().find("file size too small"), std::string::npos); } +TEST(PaimonDeletionVectorTest, RejectNullBuffer) { + DeletionVector deletion_vector; + const auto status = decode_paimon_deletion_vector_buffer(nullptr, 8, &deletion_vector); + + ASSERT_FALSE(status.ok()); + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("blob is null"), std::string::npos); +} + TEST(PaimonDeletionVectorTest, RejectLengthMismatch) { // Scenario: the big-endian length prefix protects against using a truncated or over-read DV // slice from a shared deletion-vector file. @@ -185,7 +197,8 @@ TEST(PaimonDeletionVectorTest, RejectLengthMismatch) { decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("length not match"), std::string::npos); + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("length mismatch"), std::string::npos); } TEST(PaimonDeletionVectorTest, RejectMagicMismatch) { @@ -198,7 +211,8 @@ TEST(PaimonDeletionVectorTest, RejectMagicMismatch) { decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), &deletion_vector); ASSERT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid magic number"), std::string::npos); + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("magic number mismatch"), std::string::npos); } TEST(PaimonDeletionVectorTest, RejectCorruptRoaringBitmap) { @@ -234,6 +248,40 @@ TEST(PaimonDeletionVectorTest, CacheKeyIncludesOffsetAndLength) { EXPECT_NE(first_key, build_paimon_deletion_vector_cache_key(different_length)); } +TEST(PaimonDeletionVectorTest, ValidateDescriptorRejectsInvalidRange) { + size_t bytes_read = 0; + + TPaimonDeletionFileDesc missing_path; + missing_path.__set_offset(0); + missing_path.__set_length(4); + EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(missing_path, bytes_read).ok()); + + TPaimonDeletionFileDesc deletion_file; + deletion_file.__set_path("dv.bin"); + deletion_file.__set_offset(-1); + deletion_file.__set_length(4); + EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); + + deletion_file.__set_offset(0); + deletion_file.__set_length(-1); + EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); + + deletion_file.__set_length(std::numeric_limits::max()); + EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); + + deletion_file.__set_length(MAX_PAIMON_DELETION_VECTOR_BYTES - 4); + EXPECT_TRUE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); + EXPECT_EQ(static_cast(MAX_PAIMON_DELETION_VECTOR_BYTES), bytes_read); + + deletion_file.__set_length(MAX_PAIMON_DELETION_VECTOR_BYTES - 3); + EXPECT_FALSE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); + + deletion_file.__set_offset(3); + deletion_file.__set_length(4); + EXPECT_TRUE(validate_paimon_deletion_vector_descriptor(deletion_file, bytes_read).ok()); + EXPECT_EQ(bytes_read, 8); +} + TEST(PaimonDeletionVectorTest, DecodedCacheReportsHitSeparatelyFromFileCache) { // The decoded cache lookup result is reported by ShardedKVCache itself. The creator represents // the lower File Cache/read/decode path and must only run for the miss. diff --git a/be/test/format/table/table_schema_change_helper_test.cpp b/be/test/format/table/table_schema_change_helper_test.cpp index 3fcb9d39878255..5f2b2b7cc1b1b7 100644 --- a/be/test/format/table/table_schema_change_helper_test.cpp +++ b/be/test/format/table/table_schema_change_helper_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -35,6 +36,54 @@ namespace doris { class MockTableSchemaChangeHelper : public TableSchemaChangeHelper {}; +namespace { + +schema::external::TStructField partial_name_mapping_root_field() { + TColumnType int_type; + int_type.type = TPrimitiveType::INT; + + schema::external::TStructField root_field; + for (const auto& [name, id, aliases] : + std::vector>> {{"a", 1, {"a"}}, + {"b", 2, {}}}) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_type(int_type); + field->__set_name_mapping(aliases); + field->__set_name_mapping_is_authoritative(true); + schema::external::TFieldPtr field_ptr; + field_ptr.__set_field_ptr(field); + root_field.fields.emplace_back(std::move(field_ptr)); + } + return root_field; +} + +schema::external::TStructField nested_partial_name_mapping_root_field() { + TColumnType struct_type; + struct_type.type = TPrimitiveType::STRUCT; + + auto nested_fields = partial_name_mapping_root_field(); + nested_fields.fields[0].field_ptr->__set_name_mapping({}); + + auto field = std::make_shared(); + field->__set_name("s"); + field->__set_id(10); + field->__set_type(struct_type); + field->__set_name_mapping({}); + field->__set_name_mapping_is_authoritative(true); + field->nestedField.__set_struct_field(std::move(nested_fields)); + field->__isset.nestedField = true; + + schema::external::TFieldPtr field_ptr; + field_ptr.__set_field_ptr(field); + schema::external::TStructField root_field; + root_field.fields.emplace_back(std::move(field_ptr)); + return root_field; +} + +} // namespace + TEST(PartitionColumnFillerTest, FillNullableStringPartitionValue) { SlotDescriptor slot; slot._type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true); @@ -425,6 +474,330 @@ TEST(MockTableSchemaChangeHelper, IcebergParquetNameMappingFallback) { " ScalarNode\n"); } +TEST(MockTableSchemaChangeHelper, IcebergParquetLegacyEmptyNameMappingFallsBack) { + auto root_field = partial_name_mapping_root_field(); + root_field.fields.resize(1); + root_field.fields[0].field_ptr->__set_name_mapping({}); + root_field.fields[0].field_ptr->__isset.name_mapping_is_authoritative = false; + + FieldDescriptor parquet_field; + FieldSchema file_field; + file_field.name = "a"; + file_field.field_id = -1; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetPartialNameMappingIsStrict) { + auto root_field = partial_name_mapping_root_field(); + + FieldDescriptor parquet_field; + for (const auto& name : {"a", "b"}) { + FieldSchema file_field; + file_field.name = name; + file_field.field_id = -1; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + } + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetMixedFieldIdsPreferExistingIds) { + auto root_field = partial_name_mapping_root_field(); + root_field.fields[0].field_ptr->__set_name_mapping({}); + + FieldDescriptor parquet_field; + for (const auto& [name, field_id] : + std::vector> {{"a", 1}, {"b", -1}}) { + FieldSchema file_field; + file_field.name = name; + file_field.field_id = field_id; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + } + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node, true) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetLegacyPlanFallsBackForMixedFieldIds) { + auto root_field = partial_name_mapping_root_field(); + for (auto& field : root_field.fields) { + field.field_ptr->__isset.name_mapping_is_authoritative = false; + } + + FieldDescriptor parquet_field; + for (const auto& [name, field_id] : + std::vector> {{"a", 1}, {"b", -1}}) { + FieldSchema file_field; + file_field.name = name; + file_field.field_id = field_id; + file_field.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + parquet_field._fields.emplace_back(std::move(file_field)); + } + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (file: b)\n" + " ScalarNode\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetNestedMixedFieldIdsPreferExistingIds) { + auto root_field = nested_partial_name_mapping_root_field(); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value("AAEC/w=="); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value_is_base64(true); + + FieldSchema struct_field; + struct_field.name = "s"; + struct_field.field_id = 10; + std::vector child_types; + Strings child_names; + for (const auto& [name, field_id] : + std::vector> {{"a", 1}, {"b", -1}}) { + FieldSchema child; + child.name = name; + child.field_id = field_id; + child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + child_types.emplace_back(child.data_type); + child_names.emplace_back(child.name); + struct_field.children.emplace_back(std::move(child)); + } + struct_field.data_type = std::make_shared(child_types, child_names); + + FieldDescriptor parquet_field; + parquet_field._fields.emplace_back(std::move(struct_field)); + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node, true) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); + const auto nested_node = ans_node->get_children_node("s"); + const auto default_value = nested_node->children_initial_default_value("b"); + ASSERT_TRUE(default_value.has_value()); + EXPECT_EQ(default_value->value, "AAEC/w=="); + EXPECT_TRUE(default_value->is_base64); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetLegacyPlanFallsBackForNestedMixedFieldIds) { + auto root_field = nested_partial_name_mapping_root_field(); + root_field.fields[0].field_ptr->__isset.name_mapping_is_authoritative = false; + for (auto& child : root_field.fields[0].field_ptr->nestedField.struct_field.fields) { + child.field_ptr->__isset.name_mapping_is_authoritative = false; + } + + FieldSchema struct_field; + struct_field.name = "s"; + struct_field.field_id = 10; + std::vector child_types; + Strings child_names; + for (const auto& [name, field_id] : + std::vector> {{"a", 1}, {"b", -1}}) { + FieldSchema child; + child.name = name; + child.field_id = field_id; + child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + child_types.emplace_back(child.data_type); + child_names.emplace_back(child.name); + struct_field.children.emplace_back(std::move(child)); + } + struct_field.data_type = std::make_shared(child_types, child_names); + + FieldDescriptor parquet_field; + parquet_field._fields.emplace_back(std::move(struct_field)); + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + root_field, parquet_field, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (file: b)\n" + " ScalarNode\n"); +} + +TEST(MockTableSchemaChangeHelper, + IcebergParquetDescendantIdRetainsWrapperWithAuthoritativeEmptyMapping) { + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + + auto id_field = std::make_shared(); + id_field->__set_name("id"); + id_field->__set_id(1); + id_field->__set_type(int_type); + + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_fields; + struct_fields.__set_fields({child_ptr}); + + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->__set_name_mapping({}); + struct_field->__set_name_mapping_is_authoritative(true); + struct_field->nestedField.__set_struct_field(struct_fields); + struct_field->__isset.nestedField = true; + + schema::external::TFieldPtr id_ptr; + id_ptr.__set_field_ptr(id_field); + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField table_root; + table_root.__set_fields({id_ptr, struct_ptr}); + + FieldSchema file_id; + file_id.name = "id"; + file_id.field_id = 1; + file_id.data_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_child; + file_child.name = "a"; + file_child.field_id = 2; + file_child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_struct; + file_struct.name = "s"; + file_struct.field_id = -1; + file_struct.children = {file_child}; + file_struct.data_type = std::make_shared(DataTypes {file_child.data_type}, + Strings {file_child.name}); + FieldDescriptor parquet_field; + parquet_field._fields = {file_id, file_struct}; + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + table_root, parquet_field, ans_node, true) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " id (file: id)\n" + " ScalarNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergParquetKeepsWholeFileIdModeInNestedStruct) { + TColumnType int_type; + int_type.__set_type(TPrimitiveType::INT); + TColumnType struct_type; + struct_type.__set_type(TPrimitiveType::STRUCT); + + auto id_field = std::make_shared(); + id_field->__set_name("id"); + id_field->__set_id(1); + id_field->__set_type(int_type); + auto child_field = std::make_shared(); + child_field->__set_name("a"); + child_field->__set_id(2); + child_field->__set_type(int_type); + child_field->__set_name_mapping({"legacy_a"}); + schema::external::TFieldPtr child_ptr; + child_ptr.__set_field_ptr(child_field); + schema::external::TStructField struct_children; + struct_children.__set_fields({child_ptr}); + auto struct_field = std::make_shared(); + struct_field->__set_name("s"); + struct_field->__set_id(10); + struct_field->__set_type(struct_type); + struct_field->nestedField.__set_struct_field(struct_children); + struct_field->__isset.nestedField = true; + + schema::external::TFieldPtr id_ptr; + id_ptr.__set_field_ptr(id_field); + schema::external::TFieldPtr struct_ptr; + struct_ptr.__set_field_ptr(struct_field); + schema::external::TStructField table_root; + table_root.__set_fields({id_ptr, struct_ptr}); + + FieldSchema file_id; + file_id.name = "id"; + file_id.field_id = 1; + file_id.data_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_child; + file_child.name = "legacy_a"; + file_child.field_id = -1; + file_child.data_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true); + FieldSchema file_struct; + file_struct.name = "s"; + file_struct.field_id = 10; + file_struct.children = {file_child}; + file_struct.data_type = std::make_shared(DataTypes {file_child.data_type}, + Strings {file_child.name}); + FieldDescriptor parquet_field; + parquet_field._fields = {file_id, file_struct}; + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id_with_name_mapping( + table_root, parquet_field, ans_node, true) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " id (file: id)\n" + " ScalarNode\n" + " s (file: s)\n" + " StructNode\n" + " a (not exists)\n"); +} + TEST(MockTableSchemaChangeHelper, IcebergOrcSchemaChange) { schema::external::TField test_field; TColumnType struct_type; @@ -536,6 +909,89 @@ TEST(MockTableSchemaChangeHelper, IcebergOrcNameMappingFallback) { " ScalarNode\n"); } +TEST(MockTableSchemaChangeHelper, IcebergOrcPartialNameMappingIsStrict) { + auto root_field = partial_name_mapping_root_field(); + + std::unique_ptr orc_type(orc::Type::buildTypeFromString("struct")); + std::shared_ptr ans_node; + ASSERT_TRUE( + TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, ans_node) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergOrcMixedFieldIdsPreferExistingIds) { + auto root_field = partial_name_mapping_root_field(); + root_field.fields[0].field_ptr->__set_name_mapping({}); + + std::unique_ptr orc_type(orc::Type::buildTypeFromString("struct")); + orc_type->getSubtype(0)->setAttribute(IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, "1"); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE, + ans_node, true) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); +} + +TEST(MockTableSchemaChangeHelper, IcebergOrcNestedMixedFieldIdsPreferExistingIds) { + auto root_field = nested_partial_name_mapping_root_field(); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value("AAEC/w=="); + root_field.fields[0] + .field_ptr->nestedField.struct_field.fields[1] + .field_ptr->__set_initial_default_value_is_base64(true); + + std::unique_ptr orc_type( + orc::Type::buildTypeFromString("struct>")); + const auto& attribute = IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE; + orc_type->getSubtype(0)->setAttribute(attribute, "10"); + orc_type->getSubtype(0)->getSubtype(0)->setAttribute(attribute, "1"); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), attribute, ans_node, true) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (file: s)\n" + " StructNode\n" + " a (file: a)\n" + " ScalarNode\n" + " b (not exists)\n"); + const auto nested_node = ans_node->get_children_node("s"); + const auto default_value = nested_node->children_initial_default_value("b"); + ASSERT_TRUE(default_value.has_value()); + EXPECT_EQ(default_value->value, "AAEC/w=="); + EXPECT_TRUE(default_value->is_base64); +} + +TEST(MockTableSchemaChangeHelper, IcebergOrcDoesNotBindIdlessWrapperByName) { + auto root_field = nested_partial_name_mapping_root_field(); + std::unique_ptr orc_type(orc::Type::buildTypeFromString("struct>")); + const auto& attribute = IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE; + orc_type->getSubtype(0)->getSubtype(0)->setAttribute(attribute, "1"); + + std::shared_ptr ans_node; + ASSERT_TRUE(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_field_id_with_name_mapping( + root_field, orc_type.get(), attribute, ans_node, true) + .ok()); + ASSERT_EQ(TableSchemaChangeHelper::debug(ans_node), + "StructNode\n" + " s (not exists)\n"); +} + TEST(MockTableSchemaChangeHelper, NestedMapArrayStruct) { // struct, struct>> SlotDescriptor slot1; diff --git a/be/test/format/transformer/vorc_transformer_test.cpp b/be/test/format/transformer/vorc_transformer_test.cpp new file mode 100644 index 00000000000000..4ea14766356d15 --- /dev/null +++ b/be/test/format/transformer/vorc_transformer_test.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/transformer/vorc_transformer.h" + +#include + +#include "core/block/block.h" +#include "core/column/column_string.h" +#include "core/column/column_struct.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "format/table/iceberg/schema_parser.h" +#include "io/fs/local_file_system.h" +#include "runtime/runtime_state.h" +#include "testutil/mock/mock_slot_ref.h" +#include "util/uid_util.h" + +namespace doris { + +class VOrcTransformerTest : public testing::Test { +protected: + void SetUp() override { + _file_path = "./vorc_transformer_" + UniqueId::gen_uid().to_string() + ".orc"; + _fs = io::global_local_filesystem(); + } + + void TearDown() override { static_cast(_fs->delete_file(_file_path)); } + + std::string _file_path; + std::shared_ptr _fs; +}; + +TEST_F(VOrcTransformerTest, CollectsBoundsForTopLevelFieldAfterStruct) { + auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto string_type = std::make_shared(); + VExprContextSPtrs output_exprs = + MockSlotRef::create_mock_contexts(DataTypes {struct_type, string_type}); + + const std::string schema_json = R"({ + "type": "struct", + "fields": [ + { + "id": 1, + "name": "s", + "required": true, + "type": { + "type": "struct", + "fields": [ + {"id": 2, "name": "a", "required": true, "type": "int"} + ] + } + }, + {"id": 3, "name": "b", "required": true, "type": "string"} + ] + })"; + std::unique_ptr schema = iceberg::SchemaParser::from_json(schema_json); + + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(_file_path, &file_writer).ok()); + RuntimeState state; + VOrcTransformer transformer(&state, file_writer.get(), output_exprs, "", {"s", "b"}, false, + TFileCompressType::PLAIN, schema.get(), _fs); + ASSERT_TRUE(transformer.open().ok()); + + auto nested_column = ColumnInt32::create(); + nested_column->insert_value(-1); + Columns struct_columns; + struct_columns.emplace_back(std::move(nested_column)); + auto struct_column = ColumnStruct::create(std::move(struct_columns)); + auto string_column = ColumnString::create(); + string_column->insert_data("hello", 5); + + Block block; + block.insert(ColumnWithTypeAndName(std::move(struct_column), struct_type, "s")); + block.insert(ColumnWithTypeAndName(std::move(string_column), string_type, "b")); + ASSERT_TRUE(transformer.write(block).ok()); + ASSERT_TRUE(transformer.close().ok()); + + TIcebergColumnStats stats; + ASSERT_TRUE(transformer.collect_file_statistics_after_close(&stats).ok()); + ASSERT_TRUE(stats.__isset.lower_bounds); + ASSERT_TRUE(stats.__isset.upper_bounds); + ASSERT_EQ(1, stats.lower_bounds.count(3)); + ASSERT_EQ(1, stats.upper_bounds.count(3)); + EXPECT_EQ("hello", stats.lower_bounds.at(3)); + EXPECT_EQ("hello", stats.upper_bounds.at(3)); +} + +} // namespace doris diff --git a/be/test/format/wal/wal_manager_test.cpp b/be/test/format/wal/wal_manager_test.cpp index 2189a0d33df5c3..7bdafc58cfaeec 100644 --- a/be/test/format/wal/wal_manager_test.cpp +++ b/be/test/format/wal/wal_manager_test.cpp @@ -33,6 +33,8 @@ #include "runtime/memory/mem_tracker.h" #include "runtime/runtime_state.h" #include "runtime/user_function_cache.h" +#include "util/debug_points.h" +#include "util/defer_op.h" namespace doris { @@ -382,6 +384,36 @@ TEST_F(WalManagerTest, DISABLED_read_block_fail_with_not_equal) { WARN_IF_ERROR(scanner->close(&_runtime_state), "fail to close scanner"); } +TEST_F(WalManagerTest, TestLastReplayWalFailedReason) { + const auto origin_enable_debug_points = config::enable_debug_points; + config::enable_debug_points = true; + DebugPoints::instance()->add("WalTable.replay_wals.stop"); + Defer defer([origin_enable_debug_points]() { + DebugPoints::instance()->remove("WalTable.replay_wals.stop"); + config::enable_debug_points = origin_enable_debug_points; + }); + + const int64_t wal_id = 789; + const std::string label = "test_last_replay_failed_reason"; + const std::string wal_path = _wal_dir + "/" + std::to_string(_db_id) + "/" + + std::to_string(_tb_id) + "/" + std::to_string(_version_1) + "_" + + std::to_string(_backend_id) + "_" + std::to_string(wal_id) + "_" + + label; + std::filesystem::copy_file("./be/test/exec/test_data/wal_scanner/wal_version1", wal_path, + std::filesystem::copy_options::overwrite_existing); + + WalTable wal_table(_env, _db_id, _tb_id); + wal_table.add_wal(wal_id, wal_path); + EXPECT_EQ(wal_table.replay_wals(), Status::OK()); + auto failed_reason = wal_table.get_last_replay_wal_failed_reason(); + EXPECT_NE(failed_reason.find("WalTable.replay_wals.stop"), failed_reason.npos); + EXPECT_NE(failed_reason.find(wal_path), failed_reason.npos); + + DebugPoints::instance()->remove("WalTable.replay_wals.stop"); + EXPECT_EQ(wal_table.replay_wals(), Status::OK()); + EXPECT_TRUE(wal_table.get_last_replay_wal_failed_reason().empty()); +} + TEST_F(WalManagerTest, TestDynamicWalSpaceLimt) { // 1T size_t available_bytes = 1099511627776; diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 3885deb107f072..d3428b6134a479 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -28,6 +28,7 @@ #include "common/consts.h" #include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_decimal.h" @@ -38,6 +39,7 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" +#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "exprs/vin_predicate.h" @@ -253,6 +255,81 @@ TEST(ColumnMapperDebugTest, CoversDebugStringEnumAndNestedBranches) { } } +TEST(ColumnMapperTest, ParquetRetainsIdlessComplexWrapperWithNestedFieldId) { + auto table_child = field_id_col("a", 1, i32()); + auto table_struct = struct_col("s", 10, {table_child}); + auto file_child = field_id_col("legacy_a", 1, i32(), 0); + auto file_struct = struct_name_col("s", {file_child}, 0); + + TableColumnMapper parquet_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(parquet_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(parquet_mapper.mappings().size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(parquet_mapper.mappings()[0].child_mappings.size(), 1); + ASSERT_TRUE(parquet_mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); + EXPECT_EQ(*parquet_mapper.mappings()[0].child_mappings[0].file_local_id, 0); + + TableColumnMapper orc_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(orc_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + EXPECT_FALSE(orc_mapper.mappings()[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, ParquetDescendantIdRetainsWrapperWithAuthoritativeEmptyMapping) { + auto table_struct = struct_col("s", 10, {field_id_col("a", 2, i32())}); + table_struct.has_name_mapping = true; + auto file_struct = struct_name_col("s", {field_id_col("a", 2, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 1); + EXPECT_TRUE(mapper.mappings()[0].child_mappings[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, ParquetRetainsRecursiveIdlessWrapperWithNestedFieldId) { + auto table_inner = struct_col("inner", 20, {field_id_col("leaf", 30, i32())}); + auto table_outer = struct_col("outer", 10, {table_inner}); + auto file_inner = struct_name_col("inner", {field_id_col("leaf", 30, i32(), 0)}, 0); + auto file_outer = struct_col("outer", 10, {file_inner}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .allow_idless_complex_wrapper_projection = true}); + ASSERT_TRUE(mapper.create_mapping({table_outer}, {}, {file_outer}).ok()); + + const auto& outer_mapping = mapper.mappings()[0]; + ASSERT_TRUE(outer_mapping.file_local_id.has_value()); + ASSERT_EQ(outer_mapping.child_mappings.size(), 1); + const auto& inner_mapping = outer_mapping.child_mappings[0]; + ASSERT_TRUE(inner_mapping.file_local_id.has_value()); + ASSERT_EQ(inner_mapping.child_mappings.size(), 1); + EXPECT_TRUE(inner_mapping.child_mappings[0].file_local_id.has_value()); +} + +TEST(ColumnMapperTest, MissingNestedChildRetainsBinaryInitialDefault) { + auto defaulted_child = field_id_col("data", 2, varbinary()); + defaulted_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; + defaulted_child.initial_default_value_is_base64 = true; + auto table_struct = struct_col("s", 10, {field_id_col("a", 1, i32()), defaulted_child}); + auto file_struct = struct_col("s", 10, {field_id_col("a", 1, i32(), 0)}, 0); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + ASSERT_EQ(mapper.mappings()[0].child_mappings.size(), 2); + const auto& missing = mapper.mappings()[0].child_mappings[1]; + ASSERT_TRUE(missing.initial_default_column); + Field value; + missing.initial_default_column->get(0, value); + EXPECT_EQ(value.get_type(), TYPE_VARBINARY); + EXPECT_EQ(std::string(value.get()), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); +} + void expect_mapping(const ColumnMapping& mapping, size_t global_index, const std::string& table_name, int32_t file_local_id, const std::string& file_name, const DataTypePtr& file_type, @@ -316,6 +393,38 @@ class TestFunctionExpr final : public VExpr { std::string _expr_name; }; +class ExecutableStructElementExpr final : public VExpr { +public: + explicit ExecutableStructElementExpr(DataTypePtr child_type) + : VExpr(std::move(child_type), false) { + set_node_type(TExprNodeType::FUNCTION_CALL); + TFunctionName fn_name; + fn_name.__set_function_name(_expr_name); + _fn.__set_name(fn_name); + } + + const std::string& expr_name() const override { return _expr_name; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(data_type()); + return Status::OK(); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + ColumnPtr struct_column; + RETURN_IF_ERROR( + get_child(0)->execute_column(context, block, selector, count, struct_column)); + const auto& input = assert_cast(*struct_column); + result_column = input.get_column_ptr(0); + return Status::OK(); + } + +private: + const std::string _expr_name = "element_at"; +}; + VExprSPtr table_slot(int slot_id, int column_id, DataTypePtr type, const std::string& name) { return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type), name); } @@ -340,6 +449,40 @@ VExprSPtr element_at(const VExprSPtr& parent, DataTypePtr child_type, return expr; } +VExprSPtr executable_struct_element(const VExprSPtr& parent, DataTypePtr child_type, + const std::string& child_name) { + auto expr = std::make_shared(std::move(child_type)); + expr->add_child(parent); + expr->add_child(literal(str(), Field::create_field(child_name))); + return expr; +} + +VExprSPtr executable_binary_predicate(TExprOpcode::type opcode, const VExprSPtr& left, + const VExprSPtr& right) { + const auto result_type = u8(); + TFunctionName fn_name; + fn_name.__set_function_name(opcode == TExprOpcode::GT ? "gt" : "eq"); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({left->data_type()->to_thrift(), right->data_type()->to_thrift()}); + fn.__set_ret_type(result_type->to_thrift()); + fn.__set_has_var_args(false); + + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(result_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + + auto expr = VectorizedFnCall::create_shared(node); + expr->add_child(left); + expr->add_child(right); + return expr; +} + VExprSPtr array_element_at(const VExprSPtr& parent, DataTypePtr child_type, int64_t ordinal) { auto expr = std::make_shared("element_at", std::move(child_type)); expr->add_child(parent); @@ -1928,7 +2071,8 @@ TEST(ColumnMapperConstantTest, PartitionDefaultAndVirtualColumnsUseDedicatedBran {"dt", Field::create_field("2026-06-11")}, }; - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + TableColumnMapper mapper( + {.mode = TableColumnMappingMode::BY_NAME, .enable_row_lineage_virtual_columns = true}); ASSERT_TRUE(mapper.create_mapping(table_schema, partition_values, {}).ok()); ASSERT_EQ(mapper.mappings().size(), 5); @@ -1949,7 +2093,8 @@ TEST(ColumnMapperConstantTest, PhysicalRowLineageFiltersStayFinalizeOnly) { name_col("_last_updated_sequence_number", make_nullable(i64()), 2147483539), }; - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + TableColumnMapper mapper( + {.mode = TableColumnMappingMode::BY_NAME, .enable_row_lineage_virtual_columns = true}); ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); ASSERT_EQ(mapper.mappings().size(), 2); @@ -1982,6 +2127,25 @@ TEST(ColumnMapperConstantTest, PhysicalRowLineageFiltersStayFinalizeOnly) { std::vector({2147483540, 2147483539})); } +TEST(ColumnMapperConstantTest, GenericByNameKeepsRowLineageNamesPhysical) { + const std::vector table_schema = { + name_col("_row_id", make_nullable(i64())), + name_col("_last_updated_sequence_number", make_nullable(i64())), + }; + const std::vector file_schema = { + name_col("_row_id", make_nullable(i64()), 0), + name_col("_last_updated_sequence_number", make_nullable(i64()), 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + ASSERT_EQ(mapper.mappings().size(), 2); + EXPECT_EQ(mapper.mappings()[0].virtual_column_type, TableVirtualColumnType::INVALID); + EXPECT_EQ(mapper.mappings()[0].filter_conversion, FilterConversionType::COPY_DIRECTLY); + EXPECT_EQ(mapper.mappings()[1].virtual_column_type, TableVirtualColumnType::INVALID); + EXPECT_EQ(mapper.mappings()[1].filter_conversion, FilterConversionType::COPY_DIRECTLY); +} + TEST(ColumnMapperConstantTest, MissingRowLineageDefaultExprStillUsesVirtualMapping) { auto id_column = field_id_col("id", 1, make_nullable(i32())); auto row_id_column = field_id_col("renamed_row_id", 2147483540, make_nullable(i64())); @@ -1998,7 +2162,8 @@ TEST(ColumnMapperConstantTest, MissingRowLineageDefaultExprStillUsesVirtualMappi field_id_col("name", 2, make_nullable(str()), 1), }; - TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID, + .enable_row_lineage_virtual_columns = true}); ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); ASSERT_EQ(mapper.mappings().size(), 3); @@ -2172,6 +2337,39 @@ TEST(ColumnMapperLocalizeFiltersTest, VisibleLocalFilterAddsPredicateColumnAndCo EXPECT_TRUE(localized_slot->data_type()->equals(*int_type)); } +TEST(ColumnMapperLocalizeFiltersTest, ReportsLocalizationForEachSplitMapping) { + const auto int_type = i32(); + auto table_column = name_col("id", int_type); + const std::vector table_schema = {table_column}; + TableFilter filter { + .conjunct = VExprContext::create_shared(int_gt(table_slot(0, 0, int_type, "id"), 1)), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper local_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(local_mapper.create_mapping(table_schema, {}, {name_col("id", int_type, 7)}).ok()); + FileScanRequest local_request; + FilterLocalizationResult local_result; + ASSERT_TRUE(local_mapper + .create_scan_request({filter}, table_schema, &local_request, nullptr, + &local_result) + .ok()); + ASSERT_EQ(local_result.localized_filters.size(), 1); + EXPECT_TRUE(local_result.localized_filters[0]); + ASSERT_EQ(local_request.conjuncts.size(), 1); + + TableColumnMapper missing_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(missing_mapper.create_mapping(table_schema, {}, {}).ok()); + FileScanRequest missing_request; + FilterLocalizationResult missing_result; + ASSERT_TRUE(missing_mapper + .create_scan_request({filter}, table_schema, &missing_request, nullptr, + &missing_result) + .ok()); + ASSERT_EQ(missing_result.localized_filters.size(), 1); + EXPECT_FALSE(missing_result.localized_filters[0]); + EXPECT_TRUE(missing_request.conjuncts.empty()); +} + TEST(ColumnMapperLocalizeFiltersTest, VarbinaryFilterStaysAboveFileReader) { const auto binary_type = varbinary(); const auto table_column = name_col("partition_key", binary_type); @@ -2197,6 +2395,35 @@ TEST(ColumnMapperLocalizeFiltersTest, VarbinaryFilterStaysAboveFileReader) { EXPECT_TRUE(request.conjuncts.empty()); } +TEST(ColumnMapperLocalizeFiltersTest, VarcharWidthTruncationFilterStaysAboveFileReader) { + const auto table_type = std::make_shared(3, TYPE_VARCHAR); + const auto file_type = std::make_shared(10, TYPE_VARCHAR); + const auto table_column = name_col("value", table_type); + const auto file_column = name_col("value", file_type, 7); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_column}, {}, {file_column}).ok()); + + TableFilter filter {.conjunct = VExprContext::create_shared(binary_predicate( + TExprOpcode::EQ, table_slot(0, 0, table_type, "value"), + literal(table_type, Field::create_field("abc")))), + .global_indices = {GlobalIndex(0)}}; + TQueryOptions query_options; + query_options.__set_truncate_char_or_varchar_columns(true); + RuntimeState state {query_options, TQueryGlobals()}; + FileScanRequest request; + FilterLocalizationResult localization_result; + + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_column}, &request, &state, + &localization_result) + .ok()); + ASSERT_EQ(localization_result.localized_filters.size(), 1); + EXPECT_FALSE(localization_result.localized_filters[0]); + EXPECT_TRUE(request.conjuncts.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(7)); +} + TEST(ColumnMapperLocalizeFiltersTest, NestedVarbinaryFilterStaysAboveFileReader) { const auto table_column = struct_name_col( "payload", {name_col("id", i32()), name_col("binary_value", varbinary())}); @@ -2342,10 +2569,41 @@ TEST(ColumnMapperScanRequestTest, HiddenTopLevelFilterMappingUsesNameFallback) { EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(1)})); ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(1)).is_local()); EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(1)).local_index(), LocalIndex(1)); } +TEST(ColumnMapperScanRequestTest, OrdinaryPredicateSlotRetainsOutputPayload) { + const auto int_type = i32(); + auto quantity = name_col("ss_quantity", int_type); + auto tax = name_col("ss_ext_tax", int_type); + const std::vector table_schema = {quantity, tax}; + const std::vector file_schema = { + name_col("ss_quantity", int_type, 0), + name_col("ss_ext_tax", int_type, 1), + }; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + ASSERT_EQ(mapper.mappings().size(), 2); + + auto filter_expr = int_gt(table_slot(7, 0, int_type, "ss_quantity"), 20); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, table_schema, &request).ok()); + + ASSERT_EQ(request.predicate_columns.size(), 1); + EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(0)); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(1)); + // A visible predicate slot is still part of the table output and cannot be replaced with a + // default-valued placeholder after file-local filtering. + EXPECT_TRUE(request.predicate_only_columns.empty()); +} + TEST(ColumnMapperScanRequestTest, StructOutputAndFilterOnlyChildAreMerged) { const auto int_type = i32(); const auto string_type = str(); @@ -2648,8 +2906,8 @@ TEST(ColumnMapperScanRequestTest, StructProjectionPrunesChildrenByName) { } // Scenario: a row filter reaches a struct child through an array wrapper -// (`items.item.a > 5`). The mapper keeps this as a row predicate and reads the full array root for -// predicate evaluation. +// (`items.item.a > 5`). The mapper cannot localize the filter, so it keeps the full array root in +// the lazy non-predicate set for table-level evaluation. TEST(ColumnMapperScanRequestTest, ArrayWrapperDoesNotBuildNestedPredicateFilter) { const auto int_type = i32(); const auto string_type = str(); @@ -2674,16 +2932,17 @@ TEST(ColumnMapperScanRequestTest, ArrayWrapperDoesNotBuildNestedPredicateFilter) FileScanRequest request; ASSERT_TRUE(mapper.create_scan_request({filter}, {table_array}, &request).ok()); - EXPECT_TRUE(request.non_predicate_columns.empty()); - ASSERT_EQ(request.predicate_columns.size(), 1); - EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(0)); - EXPECT_TRUE(request.predicate_columns[0].project_all_children); - EXPECT_TRUE(request.predicate_columns[0].children.empty()); + EXPECT_TRUE(request.conjuncts.empty()); + EXPECT_TRUE(request.predicate_columns.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); + EXPECT_TRUE(request.non_predicate_columns[0].project_all_children); + EXPECT_TRUE(request.non_predicate_columns[0].children.empty()); } // Scenario: a map value struct projects child `b`, while a row filter reads value child `a`. -// The filter is too complex to become a file-local nested predicate, but the predicate projection -// must replace the output projection for the same map root and contain both physical value children. +// The filter is too complex to become a file-local nested predicate. Lazy demotion must move the +// merged projection to the non-predicate set without dropping either physical value child. TEST(ColumnMapperScanRequestTest, MapFilterOnlyValueChildMergesWithOutputProjection) { const auto key_type = i32(); const auto int_type = i32(); @@ -2716,9 +2975,9 @@ TEST(ColumnMapperScanRequestTest, MapFilterOnlyValueChildMergesWithOutputProject FileScanRequest request; ASSERT_TRUE(mapper.create_scan_request({filter}, {table_map}, &request).ok()); - EXPECT_TRUE(request.non_predicate_columns.empty()); - ASSERT_EQ(request.predicate_columns.size(), 1); - const auto& projection = request.predicate_columns[0]; + EXPECT_TRUE(request.predicate_columns.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + const auto& projection = request.non_predicate_columns[0]; EXPECT_EQ(projection.column_id(), LocalColumnId(0)); ASSERT_FALSE(projection.project_all_children); ASSERT_EQ(projection.children.size(), 1); @@ -2866,6 +3125,7 @@ TEST(ColumnMapperScanRequestTest, PredicateOnlyTopLevelColumnUsesHiddenMapping) EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(10)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(10)})); EXPECT_TRUE(request.predicate_columns[0].project_all_children); EXPECT_TRUE(request.predicate_columns[0].children.empty()); @@ -2992,6 +3252,251 @@ TEST(ColumnMapperScanRequestTest, NestedElementAtConjunctUsesFileChildTypeForRen EXPECT_EQ(localized_parent_type->get_element_name(1), "bb"); } +// Scenario: Iceberg promotes a nested struct leaf from INT to BIGINT while an old file still +// stores INT. Because 15 is exactly representable as INT and every INT survives promotion to +// BIGINT, localize `s.b::BIGINT > 15::BIGINT` as `file_s.b::INT > 15::INT`. Rewriting one literal +// avoids casting every file value while preserving the table predicate exactly. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctRewritesExactLiteralToFileType) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::GT, table_leaf, + literal(table_bigint_type, Field::create_field(15))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_root = request.conjuncts[0]->root(); + ASSERT_EQ(localized_root->get_num_children(), 2); + const auto& localized_leaf = localized_root->children()[0]; + EXPECT_EQ(localized_leaf->expr_name(), "element_at"); + EXPECT_TRUE(localized_leaf->data_type()->equals(*file_int_type)); + const auto& localized_literal = localized_root->children()[1]; + EXPECT_TRUE(localized_literal->is_literal()); + EXPECT_TRUE(localized_literal->data_type()->equals(*file_int_type)); + + auto values = ColumnInt32::create(); + values->insert_value(10); + values->insert_value(20); + MutableColumns children; + children.push_back(std::move(values)); + Block block; + block.insert({ColumnStruct::create(std::move(children)), mapper.mappings()[0].file_type, "s"}); + + auto* conjunct = request.conjuncts[0].get(); + auto status = conjunct->prepare(&state, RowDescriptor()); + ASSERT_TRUE(status.ok()) << status; + status = conjunct->open(&state); + ASSERT_TRUE(status.ok()) << status; + IColumn::Filter result(block.rows(), 1); + bool can_filter_all = false; + status = conjunct->execute_filter(&block, result.data(), block.rows(), false, &can_filter_all); + ASSERT_TRUE(status.ok()) << status; + EXPECT_FALSE(can_filter_all); + EXPECT_EQ(result, IColumn::Filter({0, 1})); + conjunct->close(); +} + +// Scenario: an old file allows NULL for a nested leaf that the current table declares required. +// Although every non-NULL INT value and the literal 15 can be promoted to BIGINT exactly, the +// predicate must stay above TableReader. If `file_s.b > 15` ran first for rows [NULL, 20], it would +// discard NULL and prevent table-schema materialization from reporting the nullable-to-required +// contract violation. +TEST_F(ColumnMapperCastTest, + NestedElementAtConjunctStaysTableLevelForNullableFileLeafMappedToRequiredTableLeaf) { + const auto file_nullable_int_type = make_nullable(i32()); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_nullable_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::GT, table_leaf, + literal(table_bigint_type, Field::create_field(15))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + ASSERT_EQ(request.predicate_columns.size() + request.non_predicate_columns.size(), 1); + const auto& scan_column = request.predicate_columns.empty() ? request.non_predicate_columns[0] + : request.predicate_columns[0]; + EXPECT_EQ(scan_column.column_id(), LocalColumnId(5)); + EXPECT_TRUE(request.conjuncts.empty()); +} + +// Scenario: a narrowing file-to-table cast can produce NULL or an error for values that do not fit +// the table leaf. Evaluating that cast below TableReader can filter those rows before +// _align_column_nullability() validates the required table child. Keep the predicate at table level +// so schema materialization observes every source row first. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctStaysTableLevelForNonLosslessFileToTableCast) { + const auto file_bigint_type = i64(); + const auto table_int_type = i32(); + + auto table_a = field_id_col("a", 11, table_int_type); + auto table_struct = struct_col("s", 10, {table_a}); + auto file_a = field_id_col("a", 11, file_bigint_type, 0); + auto file_struct = struct_col("s", 10, {file_a}, 5); + + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_int_type, "a"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::EQ, table_leaf, literal(table_int_type, Field::create_field(1))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + ASSERT_EQ(request.predicate_columns.size() + request.non_predicate_columns.size(), 1); + const auto& scan_column = request.predicate_columns.empty() ? request.non_predicate_columns[0] + : request.predicate_columns[0]; + EXPECT_EQ(scan_column.column_id(), LocalColumnId(5)); + EXPECT_TRUE(request.conjuncts.empty()); +} + +// Scenario: the table literal is outside the old file leaf's INT range. Rewriting +// BIGINT 2147483648 to INT would change the predicate, so keep the literal as BIGINT and cast the +// file leaf instead: `CAST(file_s.b::INT AS BIGINT) = 2147483648::BIGINT`. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctFallsBackForOutOfRangeLiteral) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::EQ, table_leaf, + literal(table_bigint_type, Field::create_field(2147483648LL))); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_root = request.conjuncts[0]->root(); + ASSERT_EQ(localized_root->get_num_children(), 2); + const auto& localized_cast = localized_root->children()[0]; + ASSERT_NE(dynamic_cast(localized_cast.get()), nullptr); + EXPECT_TRUE(localized_cast->data_type()->equals(*table_bigint_type)); + ASSERT_EQ(localized_cast->get_num_children(), 1); + EXPECT_EQ(localized_cast->children()[0]->expr_name(), "element_at"); + EXPECT_TRUE(localized_cast->children()[0]->data_type()->equals(*file_int_type)); + EXPECT_TRUE(localized_root->children()[1]->data_type()->equals(*table_bigint_type)); +} + +// Scenario: the struct leaf is on the right side of the comparison. Literal localization must not +// depend on operand order: `15::BIGINT > s.b::BIGINT` becomes `15::INT > file_s.b::INT`. +TEST_F(ColumnMapperCastTest, NestedElementAtConjunctRewritesReverseComparisonLiteral) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto filter_expr = executable_binary_predicate( + TExprOpcode::GT, literal(table_bigint_type, Field::create_field(15)), + table_leaf); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest request; + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct}, &request, &state).ok()); + + ASSERT_EQ(request.conjuncts.size(), 1); + const auto& localized_root = request.conjuncts[0]->root(); + EXPECT_TRUE(localized_root->children()[0]->data_type()->equals(*file_int_type)); + EXPECT_TRUE(localized_root->children()[0]->is_literal()); + EXPECT_EQ(localized_root->children()[1]->expr_name(), "element_at"); + EXPECT_TRUE(localized_root->children()[1]->data_type()->equals(*file_int_type)); +} + +// Scenario: IN uses one probe type for every candidate. All exact literals may move to the INT +// file domain, but one out-of-range literal makes the complete IN predicate fall back to BIGINT. +TEST_F(ColumnMapperCastTest, NestedElementAtInPredicateUsesAllOrNothingLiteralRewrite) { + const auto file_int_type = i32(); + const auto table_bigint_type = i64(); + auto table_b = field_id_col("b", 11, table_bigint_type); + auto table_struct = struct_col("s", 10, {table_b}); + auto file_b = field_id_col("b", 11, file_int_type, 0); + auto file_struct = struct_col("s", 10, {file_b}, 5); + + const auto build_filter = [&](int64_t second_value) { + auto table_leaf = executable_struct_element( + table_slot(0, 0, table_struct.type, table_struct.name), table_bigint_type, "b"); + auto predicate = create_in_predicate(); + predicate->add_child(table_leaf); + predicate->add_child(literal(table_bigint_type, Field::create_field(10))); + predicate->add_child( + literal(table_bigint_type, Field::create_field(second_value))); + return TableFilter {.conjunct = VExprContext::create_shared(predicate), + .global_indices = {GlobalIndex(0)}}; + }; + + TableColumnMapper exact_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(exact_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest exact_request; + ASSERT_TRUE( + exact_mapper + .create_scan_request({build_filter(20)}, {table_struct}, &exact_request, &state) + .ok()); + ASSERT_EQ(exact_request.conjuncts.size(), 1); + const auto& exact_root = exact_request.conjuncts[0]->root(); + EXPECT_EQ(exact_root->children()[0]->expr_name(), "element_at"); + for (const auto& child : exact_root->children()) { + EXPECT_TRUE(child->data_type()->equals(*file_int_type)); + } + + TableColumnMapper fallback_mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(fallback_mapper.create_mapping({table_struct}, {}, {file_struct}).ok()); + FileScanRequest fallback_request; + ASSERT_TRUE(fallback_mapper + .create_scan_request({build_filter(2147483648LL)}, {table_struct}, + &fallback_request, &state) + .ok()); + ASSERT_EQ(fallback_request.conjuncts.size(), 1); + const auto& fallback_root = fallback_request.conjuncts[0]->root(); + const auto& fallback_cast = fallback_root->children()[0]; + ASSERT_NE(dynamic_cast(fallback_cast.get()), nullptr); + EXPECT_TRUE(fallback_cast->data_type()->equals(*table_bigint_type)); + EXPECT_TRUE(fallback_cast->children()[0]->data_type()->equals(*file_int_type)); + EXPECT_TRUE(fallback_root->children()[1]->data_type()->equals(*table_bigint_type)); + EXPECT_TRUE(fallback_root->children()[2]->data_type()->equals(*table_bigint_type)); +} + // Scenario: output projection reads one struct child while the row filter reads a different nested // struct child. File-local conjunct rewrite must use the merged scan projection type. In the SQL // shape below, `SELECT element_at(s, 'c') WHERE element_at(element_at(s, 'b'), 'cc') LIKE ...` @@ -3084,11 +3589,9 @@ TEST(ColumnMapperScanRequestTest, MapValuesStructChildConjunctStaysTableLevel) { ASSERT_TRUE(mapper.create_scan_request({filter}, {table_map}, &request).ok()); EXPECT_TRUE(request.conjuncts.empty()); - ASSERT_EQ(request.predicate_columns.size(), 1); - EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); - ASSERT_FALSE(request.predicate_columns[0].project_all_children); - ASSERT_EQ(request.predicate_columns[0].children.size(), 1); - EXPECT_EQ(request.predicate_columns[0].children[0].local_id(), 1); + EXPECT_TRUE(request.predicate_columns.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(1)); } // Scenario: MAP_KEYS only reads map keys, but localizing it by wrapping the evolved file map slot @@ -3127,9 +3630,9 @@ TEST(ColumnMapperScanRequestTest, MapKeysConjunctWithEvolvedValueStructStaysTabl ASSERT_TRUE(mapper.create_scan_request({filter}, {table_map}, &request).ok()); EXPECT_TRUE(request.conjuncts.empty()); - EXPECT_TRUE(request.non_predicate_columns.empty()); - ASSERT_EQ(request.predicate_columns.size(), 1); - EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + EXPECT_TRUE(request.predicate_columns.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(1)); } // Scenario: an array element struct projection only contains missing/default children; the mapper diff --git a/be/test/format_v2/delimited_text/csv_reader_test.cpp b/be/test/format_v2/delimited_text/csv_reader_test.cpp index 0ce99630c7af8b..c04e17e07f64b8 100644 --- a/be/test/format_v2/delimited_text/csv_reader_test.cpp +++ b/be/test/format_v2/delimited_text/csv_reader_test.cpp @@ -446,7 +446,7 @@ TEST_F(CsvV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(file_reader_stats.read_rows, 3); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); @@ -755,6 +755,20 @@ TEST_F(CsvV2ReaderTest, CountAggregateScansRows) { EXPECT_EQ(aggregate_result.count, 2); } +TEST_F(CsvV2ReaderTest, CountNullableColumnFallsBackToRowMaterialization) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + aggregate_request.columns.push_back( + {.projection = LocalColumnIndex::top_level(LocalColumnId(1))}); + FileAggregateResult aggregate_result; + EXPECT_TRUE(reader->get_aggregate_result(aggregate_request, &aggregate_result) + .is()); +} + // Scenario: CSV v2 parses enclosed fields itself instead of delegating to the old CsvReader. A // separator inside an enclosed string must stay inside the same CSV field. TEST_F(CsvV2ReaderTest, EnclosedFieldKeepsSeparatorInsideStringValue) { diff --git a/be/test/format_v2/delimited_text/text_reader_test.cpp b/be/test/format_v2/delimited_text/text_reader_test.cpp index f5f7309e490ff1..4927ee7622e4d4 100644 --- a/be/test/format_v2/delimited_text/text_reader_test.cpp +++ b/be/test/format_v2/delimited_text/text_reader_test.cpp @@ -41,6 +41,7 @@ #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "format_v2/column_mapper.h" +#include "format_v2/parquet/parquet_profile.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" #include "testutil/desc_tbl_builder.h" @@ -414,7 +415,7 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); EXPECT_EQ(counter_value(&_profile, "RowsReturned"), 1); @@ -423,6 +424,33 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6); } +TEST_F(TextV2ReaderTest, FormatProfilesKeepDistinctCountersInBothInitializationOrders) { + auto expect_format_children = [](RuntimeProfile* profile) { + TRuntimeProfileTree tree; + profile->to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("DelimitedTextReader")); + EXPECT_TRUE(children.at("DelimitedTextReader").contains("DelimitedRowsFilteredByConjunct")); + EXPECT_FALSE(children.at("DelimitedTextReader").contains("RowsFilteredByConjunct")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowsFilteredByConjunct")); + EXPECT_FALSE(children.at("ParquetReader").contains("DelimitedRowsFilteredByConjunct")); + }; + + RuntimeProfile parquet_first("parquet_first"); + parquet::ParquetProfile parquet_first_counters; + parquet_first_counters.init(&parquet_first); + auto parquet_first_text = create_reader(_file_path, &_params, _slots, &_state, &parquet_first); + expect_format_children(&parquet_first); + + RuntimeProfile text_first("text_first"); + auto text_first_reader = create_reader(_file_path, &_params, _slots, &_state, &text_first); + parquet::ParquetProfile text_first_parquet_counters; + text_first_parquet_counters.init(&text_first); + expect_format_children(&text_first); +} + // Scenario: Hive text has no embedded nested schema, but TableColumnMapper still needs semantic // children for complex table columns. The reader synthesizes ARRAY/MAP/STRUCT children from the // slot type while keeping the top-level local id as the text field ordinal from column_idxs. @@ -909,6 +937,20 @@ TEST_F(TextV2ReaderTest, CountAggregateScansRows) { EXPECT_EQ(aggregate_result.count, 2); } +TEST_F(TextV2ReaderTest, CountNullableColumnFallsBackToRowMaterialization) { + auto reader = create_reader(_file_path, &_params, _slots, &_state, &_profile); + auto request = std::make_shared(); + ASSERT_TRUE(reader->open(request).ok()); + + FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::COUNT; + aggregate_request.columns.push_back( + {.projection = LocalColumnIndex::top_level(LocalColumnId(1))}); + FileAggregateResult aggregate_result; + EXPECT_TRUE(reader->get_aggregate_result(aggregate_request, &aggregate_result) + .is()); +} + // Scenario: a non-first split starts inside a text record and must skip the partial first line. TEST_F(TextV2ReaderTest, NonFirstSplitSkipsPartialFirstRecord) { const auto split_path = (_test_dir / "split.text").string(); diff --git a/be/test/format_v2/jni/jdbc_reader_test.cpp b/be/test/format_v2/jni/jdbc_reader_test.cpp new file mode 100644 index 00000000000000..5763a6deb00895 --- /dev/null +++ b/be/test/format_v2/jni/jdbc_reader_test.cpp @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/jni/jdbc_reader.h" + +#include + +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" + +namespace doris::format::jdbc { + +TEST(JdbcJniReaderTest, NonNullableSpecialTypeRejectsCastNull) { + auto data = ColumnString::create(); + data->insert_default(); + auto null_map = ColumnUInt8::create(); + null_map->get_data().push_back(1); + auto result = ColumnNullable::create(std::move(data), std::move(null_map)); + + EXPECT_TRUE(validate_non_nullable_special_type_result(*result, 1) + .is()); +} + +TEST(JdbcJniReaderTest, NonNullableSpecialTypeAcceptsSuccessfulCast) { + auto data = ColumnString::create(); + data->insert_data("ok", 2); + auto null_map = ColumnUInt8::create(); + null_map->get_data().push_back(0); + auto result = ColumnNullable::create(std::move(data), std::move(null_map)); + + EXPECT_TRUE(validate_non_nullable_special_type_result(*result, 1).ok()); +} + +} // namespace doris::format::jdbc diff --git a/be/test/format_v2/jni/jni_table_reader_test.cpp b/be/test/format_v2/jni/jni_table_reader_test.cpp index db3ec3c7e02b4d..95eda4a557e790 100644 --- a/be/test/format_v2/jni/jni_table_reader_test.cpp +++ b/be/test/format_v2/jni/jni_table_reader_test.cpp @@ -19,10 +19,12 @@ #include +#include #include #include #include #include +#include #include #include "io/io_common.h" @@ -40,9 +42,15 @@ class FakeJniTableReader final : public JniTableReader { std::vector propagated_batch_sizes; std::vector close_results; bool next_eof = false; + std::chrono::milliseconds init_delay {0}; + std::chrono::milliseconds open_delay {0}; + std::chrono::milliseconds close_delay {0}; protected: - std::string connector_class() const override { return "test/FakeJniScanner"; } + std::string connector_class() const override { + std::this_thread::sleep_for(init_delay); + return "test/FakeJniScanner"; + } Status build_scanner_params(std::map* params) const override { params->clear(); @@ -57,6 +65,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _close_jni_scanner() override { + std::this_thread::sleep_for(close_delay); if (!TEST_scanner_opened()) { return Status::OK(); } @@ -80,6 +89,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _open_jni_scanner() override { + std::this_thread::sleep_for(open_delay); open_batch_sizes.push_back(TEST_batch_size()); TEST_set_split_state(true, false); return Status::OK(); @@ -179,6 +189,7 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { .conjuncts = std::nullopt, .partition_prune_conjuncts = {}, .all_runtime_filters_applied = true, + .condition_cache_digest = std::nullopt, .cache = nullptr, .current_range = {}, .current_split_format = FileFormat::JNI, @@ -190,5 +201,36 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { EXPECT_TRUE(reader.TEST_scanner_opened()); } +TEST(JniTableReaderTest, CommonLifecycleTimersContainJniLifecycleWork) { + constexpr auto delay = std::chrono::milliseconds(8); + RuntimeProfile profile("JniLifecycleContainment"); + FakeJniTableReader reader; + reader.init_delay = delay; + ASSERT_TRUE(init_reader(&reader, nullptr, &profile).ok()); + ASSERT_GE(profile.get_counter("InitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.open_delay = delay; + ASSERT_TRUE(reader.prepare_split({ + .partition_values = {}, + .conjuncts = std::nullopt, + .partition_prune_conjuncts = {}, + .all_runtime_filters_applied = true, + .condition_cache_digest = std::nullopt, + .cache = nullptr, + .current_range = {}, + .current_split_format = FileFormat::JNI, + .global_rowid_context = std::nullopt, + }) + .ok()); + ASSERT_GE(profile.get_counter("PrepareSplitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.close_delay = delay; + ASSERT_TRUE(reader.close().ok()); + EXPECT_GE(profile.get_counter("CloseTime")->value(), + std::chrono::duration_cast(delay).count()); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/json/json_reader_test.cpp b/be/test/format_v2/json/json_reader_test.cpp index 683a47dee299cc..e8d9aec40af34a 100644 --- a/be/test/format_v2/json/json_reader_test.cpp +++ b/be/test/format_v2/json/json_reader_test.cpp @@ -173,14 +173,21 @@ struct ReadResult { bool eof = false; size_t second_rows = 0; bool second_eof = false; + size_t document_buffer_size = 0; std::vector schema; }; ReadResult read_once(const std::string& file_name, const std::string& content, TFileScanRangeParams params, const std::vector& slots, - const std::vector& requested_local_ids, bool read_twice = false) { + const std::vector& requested_local_ids, bool read_twice = false, + bool is_hive_table = false) { const auto file_path = write_json_file(file_name, content); auto range = file_range(file_path); + if (is_hive_table) { + TTableFormatFileDesc table_format; + table_format.__set_table_format_type("hive"); + range.__set_table_format_params(std::move(table_format)); + } auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; @@ -210,6 +217,7 @@ ReadResult read_once(const std::string& file_name, const std::string& content, result.block = make_block(result.schema, requested_local_ids); result.status = reader.get_block(&result.block, &result.rows, &result.eof); + result.document_buffer_size = reader.TEST_document_buffer_size(); if (result.status.ok() && read_twice) { auto eof_block = make_block(result.schema, requested_local_ids); result.second_status = @@ -318,6 +326,20 @@ TEST(JsonReaderTest, ReadsRequestedColumnsInFileScanRequestOrder) { ASSERT_TRUE(result.second_status.ok()) << result.second_status.to_string(); EXPECT_EQ(result.second_rows, 0); EXPECT_TRUE(result.second_eof); + EXPECT_EQ(result.document_buffer_size, 0); +} + +TEST(JsonReaderTest, HiveColumnLookupIsCaseInsensitiveWithoutNormalizedKeys) { + ObjectPool pool; + auto slots = build_slots(&pool); + auto result = read_once("hive_case.jsonl", + R"({"ID":7,"NaMe":"alice"})" + "\n", + json_scan_params(), slots, {0, 1}, false, true); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.rows, 1); + EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 0), 7); + EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 0), "alice"); } TEST(JsonReaderTest, ReadsSingleDocumentOuterArray) { diff --git a/be/test/format_v2/native/native_reader_test.cpp b/be/test/format_v2/native/native_reader_test.cpp index aaa7aa90e0681e..2745ecf852c38b 100644 --- a/be/test/format_v2/native/native_reader_test.cpp +++ b/be/test/format_v2/native/native_reader_test.cpp @@ -295,7 +295,7 @@ TEST(NativeV2ReaderTest, RejectsInvalidHeaderAndEmptyFile) { static_cast(io::global_local_filesystem()->delete_file(empty_path)); } -TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndHeaderOnlyFile) { +TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndReportsHeaderOnlyFileAsEmpty) { std::filesystem::create_directories("./log"); RuntimeState state; RuntimeProfile profile("native_v2_reader_header_boundary_test"); @@ -322,7 +322,8 @@ TEST(NativeV2ReaderTest, RejectsUnsupportedVersionAndHeaderOnlyFile) { auto header_only_reader = create_reader(header_only_path, &state, &profile); ASSERT_TRUE(header_only_reader->init(&state).ok()); std::vector schema; - EXPECT_FALSE(header_only_reader->get_schema(&schema).ok()); + const auto header_only_status = header_only_reader->get_schema(&schema); + EXPECT_TRUE(header_only_status.is()) << header_only_status; static_cast(io::global_local_filesystem()->delete_file(header_only_path)); } @@ -350,6 +351,34 @@ TEST(NativeV2ReaderTest, RejectsTruncatedBlockDuringSchemaProbe) { static_cast(io::global_local_filesystem()->delete_file(path)); } +TEST(NativeV2ReaderTest, RejectsPartialBlockLengthAsCorruption) { + std::filesystem::create_directories("./log"); + RuntimeState state; + RuntimeProfile profile("native_v2_reader_partial_length_test"); + + std::string header; + header.append(DORIS_NATIVE_MAGIC, sizeof(DORIS_NATIVE_MAGIC)); + uint8_t version_buffer[sizeof(uint32_t)]; + encode_fixed32_le(version_buffer, DORIS_NATIVE_FORMAT_VERSION); + header.append(reinterpret_cast(version_buffer), sizeof(version_buffer)); + + for (size_t prefix_bytes = 1; prefix_bytes < sizeof(uint64_t); ++prefix_bytes) { + const auto path = "./log/native_v2_partial_length_" + std::to_string(prefix_bytes) + "_" + + UniqueId::gen_uid().to_string() + ".native"; + auto content = header; + content.append(prefix_bytes, '\0'); + ASSERT_TRUE(write_file(path, content).ok()); + + auto reader = create_reader(path, &state, &profile); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + const auto status = reader->get_schema(&schema); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("truncated native block length"), std::string::npos); + static_cast(io::global_local_filesystem()->delete_file(path)); + } +} + TEST(NativeV2ReaderTest, RejectsZeroLengthBlockAndInvalidPBlock) { std::filesystem::create_directories("./log"); RuntimeState state; @@ -374,7 +403,9 @@ TEST(NativeV2ReaderTest, RejectsZeroLengthBlockAndInvalidPBlock) { auto zero_len_reader = create_reader(zero_len_path, &state, &profile); ASSERT_TRUE(zero_len_reader->init(&state).ok()); std::vector schema; - EXPECT_FALSE(zero_len_reader->get_schema(&schema).ok()); + const auto zero_len_status = zero_len_reader->get_schema(&schema); + EXPECT_TRUE(zero_len_status.is()) << zero_len_status; + EXPECT_NE(zero_len_status.to_string().find("zero-length native block"), std::string::npos); static_cast(io::global_local_filesystem()->delete_file(zero_len_path)); const auto invalid_pblock_path = diff --git a/be/test/format_v2/orc/orc_file_input_stream_test.cpp b/be/test/format_v2/orc/orc_file_input_stream_test.cpp index 54c89505edc3f5..8d63a4ab21ae65 100644 --- a/be/test/format_v2/orc/orc_file_input_stream_test.cpp +++ b/be/test/format_v2/orc/orc_file_input_stream_test.cpp @@ -329,14 +329,49 @@ TEST(OrcFileInputStreamTest, PublishesClusterProfileExactlyOnce) { input.beforeReadStripe( std::make_unique(200, std::vector {}), selected_columns({}), next_streams); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); } - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); +} + +TEST(OrcFileInputStreamTest, MergedIoChildrenStayIsolatedInBothInitializationOrders) { + for (const bool generic_first : {false, true}) { + auto reader = std::make_shared(256); + RuntimeProfile profile(generic_first ? "generic_first" : "orc_first"); + auto register_generic = [&] { + ADD_TIMER(&profile, "MergedSmallIO"); + return ADD_CHILD_COUNTER_WITH_LEVEL(&profile, "RequestIO", TUnit::UNIT, "MergedSmallIO", + 1); + }; + RuntimeProfile::Counter* generic_request_io = nullptr; + if (generic_first) { + generic_request_io = register_generic(); + } + { + OrcFileInputStream input("test.orc", reader, nullptr, &profile, + {.once_max_read_bytes = 16, .max_merge_distance_bytes = 0}); + if (!generic_first) { + generic_request_io = register_generic(); + } + StripeStreamMap streams; + input.beforeReadStripe( + std::make_unique( + 100, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams); + std::array data {}; + find_stream(streams, 1, ::orc::StreamKind_DATA)->read(data.data(), data.size(), 100); + } + ASSERT_NE(generic_request_io, nullptr); + EXPECT_EQ(generic_request_io->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 1); + } } } // namespace diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index b98e2bf14e6d0e..63a09b04b1cced 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -29,17 +29,20 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include #include #include "common/config.h" +#include "common/exception.h" #include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_array.h" @@ -70,6 +73,7 @@ #include "format_v2/expr/cast.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "gen_cpp/Types_types.h" #include "io/fs/buffered_reader.h" #include "io/io_common.h" @@ -77,11 +81,30 @@ #include "runtime/runtime_state.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/debug_points.h" #include "util/timezone_utils.h" namespace doris { namespace { +class ScopedDebugPoint { +public: + ScopedDebugPoint(std::string name, std::function callback) + : _name(std::move(name)), _enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add_with_callback(_name, std::move(callback)); + } + + ~ScopedDebugPoint() { + DebugPoints::instance()->remove(_name); + config::enable_debug_points = _enable_debug_points; + } + +private: + std::string _name; + bool _enable_debug_points; +}; + std::filesystem::path unique_test_dir(std::string_view prefix) { static std::atomic next_dir_id {0}; std::string name(prefix); @@ -337,6 +360,21 @@ class NullableInt32GreaterThanExpr final : public VExpr { const std::string _expr_name = "NullableInt32GreaterThanExpr"; }; +class FailingRowFilterExpr final : public VExpr { +public: + FailingRowFilterExpr() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InvalidArgument("synthetic row filter failure"); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const std::string _expr_name = "FailingRowFilterExpr"; +}; + class NullableInt32LessThanExpr final : public VExpr { public: NullableInt32LessThanExpr(int column_id, int32_t value) @@ -582,9 +620,7 @@ class NullableInt32IsNullExpr final : public VExpr { class CompoundPredicateExpr final : public VExpr { public: CompoundPredicateExpr(TExprOpcode::type opcode, VExprSPtrs children) - : VExpr(std::make_shared(), false), - _expr_name(opcode == TExprOpcode::COMPOUND_OR ? "CompoundOrPredicateExpr" - : "CompoundNotPredicateExpr") { + : VExpr(std::make_shared(), false), _expr_name("CompoundPredicateExpr") { _node_type = TExprNodeType::COMPOUND_PRED; _opcode = opcode; set_children(std::move(children)); @@ -607,7 +643,7 @@ class CompoundPredicateExpr final : public VExpr { return Status::OK(); } - DORIS_CHECK(_opcode == TExprOpcode::COMPOUND_OR); + DORIS_CHECK(_opcode == TExprOpcode::COMPOUND_AND || _opcode == TExprOpcode::COMPOUND_OR); DORIS_CHECK(!_children.empty()); std::vector child_columns; child_columns.reserve(_children.size()); @@ -617,12 +653,17 @@ class CompoundPredicateExpr final : public VExpr { child_columns.push_back(std::move(child_column)); } for (size_t row = 0; row < count; ++row) { - result_data[row] = 0; + result_data[row] = _opcode == TExprOpcode::COMPOUND_AND; for (const auto& child_column : child_columns) { - if (bool_value(*child_column, row)) { + const auto child_value = bool_value(*child_column, row); + if (_opcode == TExprOpcode::COMPOUND_OR && child_value) { result_data[row] = 1; break; } + if (_opcode == TExprOpcode::COMPOUND_AND && !child_value) { + result_data[row] = 0; + break; + } } } result_column = std::move(result); @@ -674,6 +715,22 @@ class RuntimeFilterWrapperExpr final : public VExpr { const std::string _expr_name; }; +class CountingIntHybridSet final : public HybridSet { +public: + using HybridSet::HybridSet; + + HybridSetBase::IteratorBase* begin() override { + ++_begin_calls; + return HybridSet::begin(); + } + + void reset_begin_calls() { _begin_calls = 0; } + size_t begin_calls() const { return _begin_calls; } + +private: + size_t _begin_calls = 0; +}; + class NullableInt32CastToInt64GreaterThanExpr final : public VExpr { public: NullableInt32CastToInt64GreaterThanExpr(int column_id, int64_t value) @@ -4274,7 +4331,8 @@ void write_two_stripe_constant_date_file(const std::string& file_path, int64_t f void write_two_stripe_constant_timestamp_file(const std::string& file_path, int64_t first_timestamp_second, - int64_t second_timestamp_second) { + int64_t second_timestamp_second, + std::string_view writer_timezone = "GMT") { auto type = std::unique_ptr<::orc::Type>( ::orc::Type::buildTypeFromString("struct")); @@ -4282,6 +4340,7 @@ void write_two_stripe_constant_timestamp_file(const std::string& file_path, ::orc::WriterOptions options; options.setCompression(::orc::CompressionKind_NONE); options.setMemoryPool(::orc::getDefaultPool()); + options.setTimezoneName(std::string(writer_timezone)); options.setStripeSize(1); options.setDictionaryKeySizeThreshold(0); auto writer = ::orc::createWriter(*type, &memory_stream, options); @@ -4358,6 +4417,40 @@ std::vector get_orc_stripe_layout(const std::string& file_path) return layout; } +bool corrupt_orc_stripe_statistics(const std::string& file_path) { + std::ifstream in(file_path, std::ios::binary | std::ios::ate); + const auto file_size = in.tellg(); + if (file_size <= 0) { + return false; + } + in.seekg(0); + std::vector data(static_cast(file_size)); + in.read(data.data(), static_cast(data.size())); + if (!in.good()) { + return false; + } + + ::orc::ReaderOptions options; + options.setMemoryPool(*::orc::getDefaultPool()); + auto input_stream = std::make_unique(data.data(), data.size()); + auto reader = ::orc::createReader(std::move(input_stream), options); + const auto metadata_length = reader->getStripeStatisticsLength(); + const auto footer_length = reader->getFileFooterLength(); + const auto postscript_length = reader->getFilePostscriptLength(); + const auto orc_file_length = reader->getFileLength(); + const auto trailing_length = metadata_length + footer_length + postscript_length + 1; + if (metadata_length == 0 || orc_file_length != data.size() || + trailing_length > orc_file_length) { + return false; + } + + const auto metadata_offset = orc_file_length - trailing_length; + std::fill_n(data.data() + metadata_offset, metadata_length, static_cast(0xff)); + std::ofstream out(file_path, std::ios::binary | std::ios::trunc); + out.write(data.data(), static_cast(data.size())); + return out.good(); +} + Block build_file_block(const std::vector& schema) { Block block; for (const auto& field : schema) { @@ -4393,6 +4486,21 @@ format::LocalColumnIndex struct_child_projection(int32_t root_field_id, int32_t class NewOrcReaderTest : public testing::Test { protected: + enum class DirectInPredicateShape { + ROOT, + AND, + OR, + OR_WITH_NULL_SAFE_EQUAL, + }; + + struct DirectInScanResult { + std::vector ids; + size_t materialize_calls = 0; + size_t prepare_literals_calls = 0; + int64_t filtered_row_groups = 0; + int64_t filtered_row_groups_by_min_max = 0; + }; + void SetUp() override { _test_dir = unique_test_dir("doris_new_orc_reader_test"); std::filesystem::remove_all(_test_dir); @@ -4447,6 +4555,88 @@ class NewOrcReaderTest : public testing::Test { nullptr, profile, std::nullopt); } + Status scan_direct_in_filter(const std::string& file_path, int32_t in_list_size, + bool enable_sarg, int max_pushdown_conditions_per_column, + DirectInPredicateShape shape, DirectInScanResult* result) const { + DORIS_CHECK(result != nullptr); + *result = {}; + + auto reader = create_reader_for_path(file_path); + TQueryOptions query_options; + query_options.__set_enable_orc_filter_by_min_max(enable_sarg); + query_options.__set_max_pushdown_conditions_per_column(max_pushdown_conditions_per_column); + RuntimeState state {query_options, TQueryGlobals()}; + RETURN_IF_ERROR(reader->init(&state)); + + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + DORIS_CHECK(schema.size() == 2); + + constexpr int32_t in_list_start = 1000; + auto filter = std::make_shared(false); + for (int32_t value = in_list_start; value < in_list_start + in_list_size; ++value) { + filter->insert(&value); + } + + auto node = make_filter_in_node(TExprNodeType::IN_PRED); + auto direct_in = VDirectInPredicate::create_shared(node, filter, true); + direct_in->add_child(TableSlotRef::create_shared(0, 0, -1, schema[0].type, "id")); + VExprSPtr predicate = RuntimeFilterExpr::create_shared(node, direct_in, 0.0, false, 7); + if (shape == DirectInPredicateShape::AND) { + predicate = std::make_shared( + TExprOpcode::COMPOUND_AND, + VExprSPtrs {std::move(predicate), + std::make_shared(0, 500)}); + } else if (shape == DirectInPredicateShape::OR) { + predicate = std::make_shared( + TExprOpcode::COMPOUND_OR, + VExprSPtrs {std::move(predicate), + std::make_shared(0, 0)}); + } else if (shape == DirectInPredicateShape::OR_WITH_NULL_SAFE_EQUAL) { + predicate = std::make_shared( + TExprOpcode::COMPOUND_OR, + VExprSPtrs {std::move(predicate), + std::make_shared(0, -1, + false)}); + } else { + DORIS_CHECK(shape == DirectInPredicateShape::ROOT); + } + auto context = VExprContext::create_shared(std::move(predicate)); + RETURN_IF_ERROR(context->prepare(&state, RowDescriptor())); + RETURN_IF_ERROR(context->open(&state)); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(std::move(context)); + + ScopedDebugPoint debug_point("OrcSearchArgument.make_in_literals_for_sarg", + [&]() { ++result->prepare_literals_calls; }); + filter->reset_begin_calls(); + RETURN_IF_ERROR(reader->open(request)); + + bool eof = false; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + RETURN_IF_ERROR(reader->get_block(&block, &rows, &eof)); + if (rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + result->ids.push_back(ids.get_element(row)); + } + } + + result->materialize_calls = filter->begin_calls(); + result->filtered_row_groups = reader->reader_statistics().filtered_row_groups; + result->filtered_row_groups_by_min_max = + reader->reader_statistics().filtered_row_groups_by_min_max; + return Status::OK(); + } + std::filesystem::path _test_dir; std::string _file_path; }; @@ -4641,13 +4831,48 @@ TEST_F(NewOrcReaderTest, AggregatePushdownUsesPrunedStripes) { EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1); } -TEST_F(NewOrcReaderTest, AggregatePushdownTimestampMinMaxUsesSessionTimezone) { +TEST_F(NewOrcReaderTest, AggregatePushdownTimestampMinMaxMatchesScanInSessionTimezone) { const auto multi_stripe_file_path = (_test_dir / "aggregate_timestamp_timezone.orc").string(); - write_two_stripe_constant_timestamp_file(multi_stripe_file_path, 0, 3600); + write_two_stripe_constant_timestamp_file(multi_stripe_file_path, 1609430400, 1609434000, + "Asia/Shanghai"); - auto reader = create_reader_for_path(multi_stripe_file_path); RuntimeState state {TQueryOptions(), TQueryGlobals()}; state.set_timezone("Asia/Shanghai"); + + auto scan_reader = create_reader_for_path(multi_stripe_file_path); + ASSERT_TRUE(scan_reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(scan_reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + auto scan_request = std::make_shared(); + scan_request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(scan_reader->open(scan_request).ok()); + + std::optional> scan_min; + std::optional> scan_max; + size_t scan_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(scan_reader->get_block(&block, &rows, &eof).ok()); + scan_rows += rows; + const auto& nullable = assert_cast(*block.get_by_position(0).column); + const auto& values = assert_cast(nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + ASSERT_FALSE(nullable.is_null_at(row)); + const auto value = values.get_element(row); + scan_min = !scan_min.has_value() || value < *scan_min ? value : *scan_min; + scan_max = !scan_max.has_value() || *scan_max < value ? value : *scan_max; + } + } + ASSERT_EQ(scan_rows, 400); + ASSERT_TRUE(scan_min.has_value()); + ASSERT_TRUE(scan_max.has_value()); + EXPECT_EQ(*scan_min, make_datetime_v2(2021, 1, 1, 0, 0, 0, 123000)); + EXPECT_EQ(*scan_max, make_datetime_v2(2021, 1, 1, 1, 0, 0, 123000)); + + auto reader = create_reader_for_path(multi_stripe_file_path); ASSERT_TRUE(reader->init(&state).ok()); auto request = std::make_shared(); @@ -4665,10 +4890,87 @@ TEST_F(NewOrcReaderTest, AggregatePushdownTimestampMinMaxUsesSessionTimezone) { EXPECT_EQ(aggregate_result.count, 400); EXPECT_TRUE(aggregate_result.columns[0].has_min); EXPECT_TRUE(aggregate_result.columns[0].has_max); - EXPECT_EQ(aggregate_result.columns[0].min_value.get(), - make_datetime_v2(1970, 1, 1, 8, 0, 0, 123000)); - EXPECT_EQ(aggregate_result.columns[0].max_value.get(), - make_datetime_v2(1970, 1, 1, 9, 0, 0, 123000)); + EXPECT_EQ(aggregate_result.columns[0].min_value.get(), *scan_min); + EXPECT_EQ(aggregate_result.columns[0].max_value.get(), *scan_max); +} + +TEST_F(NewOrcReaderTest, AggregatePushdownTimestampMinMaxFallsBackForPreOrc135Writer) { + const auto file_path = find_repo_file( + "contrib/apache-orc/java/core/src/test/resources/orc-file-no-timezone.orc"); + ASSERT_TRUE(std::filesystem::exists(file_path)) << file_path; + + std::ifstream in(file_path, std::ios::binary | std::ios::ate); + ASSERT_TRUE(in.good()); + const auto file_size = in.tellg(); + ASSERT_GT(file_size, 0); + in.seekg(0); + std::vector data(static_cast(file_size)); + in.read(data.data(), static_cast(data.size())); + ASSERT_TRUE(in.good()); + + ::orc::ReaderOptions options; + options.setMemoryPool(*::orc::getDefaultPool()); + auto input_stream = std::make_unique(data.data(), data.size()); + auto metadata_reader = ::orc::createReader(std::move(input_stream), options); + ASSERT_EQ(metadata_reader->getWriterVersion(), ::orc::WriterVersion_HIVE_8732); + auto stripe_statistics = metadata_reader->getStripeStatistics(0); + ASSERT_NE(stripe_statistics, nullptr); + const auto* timestamp_statistics = dynamic_cast( + stripe_statistics->getColumnStatistics(1)); + ASSERT_NE(timestamp_statistics, nullptr); + ASSERT_TRUE(timestamp_statistics->hasMinimum()); + ASSERT_TRUE(timestamp_statistics->hasMaximum()); + + auto reader = create_reader_for_path(file_path.string()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("UTC"); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + aggregate_request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(0))}); + format::FileAggregateResult aggregate_result; + const auto aggregate_status = + reader->get_aggregate_result(aggregate_request, &aggregate_result); + ASSERT_TRUE(aggregate_status.is()) << aggregate_status; + + const auto read_values = [&](format::orc::OrcReader* scan_reader, + std::vector* values) -> Status { + DORIS_CHECK(scan_reader != nullptr); + DORIS_CHECK(values != nullptr); + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + RETURN_IF_ERROR(scan_reader->get_block(&block, &rows, &eof)); + for (size_t row = 0; row < rows; ++row) { + values->push_back(schema[0].type->to_string(*block.get_by_position(0).column, row)); + } + } + return Status::OK(); + }; + + std::vector fallback_values; + ASSERT_TRUE(read_values(reader.get(), &fallback_values).ok()); + + auto baseline_reader = create_reader_for_path(file_path.string()); + ASSERT_TRUE(baseline_reader->init(&state).ok()); + auto baseline_request = std::make_shared(); + baseline_request->non_predicate_columns = {field_projection(0)}; + ASSERT_TRUE(baseline_reader->open(baseline_request).ok()); + + std::vector baseline_values; + ASSERT_TRUE(read_values(baseline_reader.get(), &baseline_values).ok()); + ASSERT_EQ(baseline_values.size(), 2); + EXPECT_EQ(fallback_values, baseline_values); } TEST_F(NewOrcReaderTest, AggregatePushdownTimestampInstantMinMaxUsesTimestampTzWhenMapped) { @@ -4976,16 +5278,17 @@ TEST_F(NewOrcReaderTest, StripePrefetchPublishesMergedReadProfile) { } ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - ASSERT_NE(profile.get_counter("ApplyBytes"), nullptr); - ASSERT_NE(profile.get_counter("ClusterNum"), nullptr); - ASSERT_NE(profile.get_counter("OverReadBytes"), nullptr); - EXPECT_GT(profile.get_counter("RequestIO")->value(), profile.get_counter("MergedIO")->value()); - EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); - EXPECT_GT(profile.get_counter("ApplyBytes")->value(), 0); - EXPECT_GT(profile.get_counter("ClusterNum")->value(), 0); - EXPECT_GE(profile.get_counter("OverReadBytes")->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedApplyBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedClusterNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedOverReadBytes"), nullptr); + EXPECT_GT(profile.get_counter("OrcMergedRequestIO")->value(), + profile.get_counter("OrcMergedIO")->value()); + EXPECT_GT(profile.get_counter("OrcMergedIO")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedApplyBytes")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedClusterNum")->value(), 0); + EXPECT_GE(profile.get_counter("OrcMergedOverReadBytes")->value(), 0); } TEST_F(NewOrcReaderTest, StripePrefetchCanBeDisabledByZeroOnceMaxReadBytes) { @@ -5376,6 +5679,29 @@ TEST_F(NewOrcReaderTest, ConjunctFiltersRows) { EXPECT_EQ(values.get_data_at(2).to_string(), "five"); } +TEST_F(NewOrcReaderTest, RowFilterFailureRetainsNextBatchContext) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->non_predicate_columns = {field_projection(1)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared())); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_NE(status.to_string().find("nextBatch failed"), std::string::npos) << status; + EXPECT_NE(status.to_string().find("synthetic row filter failure"), std::string::npos) << status; +} + TEST_F(NewOrcReaderTest, OrcLazyDecodesOnlySelectedNonPredicateRows) { auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -5436,8 +5762,8 @@ TEST_F(NewOrcReaderTest, ClosePublishesOrcLazyStatisticsToRuntimeProfile) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 2); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 2); } TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { @@ -5466,8 +5792,8 @@ TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); } TEST_F(NewOrcReaderTest, ConditionCacheMissMarksSurvivingGranules) { @@ -6415,13 +6741,13 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { ASSERT_EQ(result_rows, 200); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsReadNum"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByGroup"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - ASSERT_NE(profile.get_counter("FilteredBytes"), nullptr); - ASSERT_NE(profile.get_counter("FileNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsReadNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByGroup"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcFileNum"), nullptr); const std::array orc_reader_metric_counters { "ReaderCall", "ReaderInclusiveLatencyUs", @@ -6440,19 +6766,63 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { for (const auto counter_name : orc_reader_metric_counters) { ASSERT_NE(profile.get_counter(std::string(counter_name)), nullptr) << counter_name; } - EXPECT_EQ(profile.get_counter("RowGroupsFiltered")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsReadNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFiltered")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsReadNum")->value(), 1); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 1); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 3); - EXPECT_EQ(profile.get_counter("FilteredRowsByGroup")->value(), 400); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); - EXPECT_GT(profile.get_counter("FilteredBytes")->value(), 0); - EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByGroup")->value(), 400); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); + EXPECT_GT(profile.get_counter("OrcFilteredBytes")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcFileNum")->value(), 1); EXPECT_EQ(profile.get_counter("ReadRowCount")->value(), static_cast(file_reader_stats.read_rows)); } +TEST_F(NewOrcReaderTest, ProfileKeepsPruningCountersBelowOrcReader) { + RuntimeProfile profile("new_orc_reader_hierarchy_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("OrcReader")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsFiltered")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsReadNum")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcFilteredRowsByGroup")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenParquetInitializesFirst) { + RuntimeProfile profile("parquet_then_orc_profile"); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenOrcInitializesFirst) { + RuntimeProfile profile("orc_then_parquet_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { const auto multi_stripe_file_path = (_test_dir / "profile_minmax_disabled.orc").string(); write_multi_stripe_orc_int_file(multi_stripe_file_path, {1, 1000, 2000}); @@ -6489,10 +6859,10 @@ TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { ASSERT_NE(profile.get_counter("SelectedRowGroupCount"), nullptr); ASSERT_NE(profile.get_counter("EvaluatedRowGroupCount"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 0); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 0); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 0); } TEST_F(NewOrcReaderTest, SargConjunctPrunesStripes) { @@ -6632,6 +7002,99 @@ TEST_F(NewOrcReaderTest, SargRuntimeFilterWrapperConjunctPrunesStripes) { EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); } +TEST_F(NewOrcReaderTest, SargDirectInCompoundMaterializesLiteralsOnce) { + const auto multi_stripe_file_path = (_test_dir / "sarg_direct_in_prepared_once.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + constexpr int32_t in_list_size = 1024; + for (const auto shape : {DirectInPredicateShape::AND, DirectInPredicateShape::OR}) { + SCOPED_TRACE(shape == DirectInPredicateShape::AND ? "AND" : "OR"); + DirectInScanResult enabled; + DirectInScanResult disabled; + ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size, true, in_list_size, + shape, &enabled) + .ok()); + ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size, false, in_list_size, + shape, &disabled) + .ok()); + + EXPECT_EQ(enabled.materialize_calls, 1); + EXPECT_EQ(enabled.prepare_literals_calls, 1); + EXPECT_EQ(enabled.filtered_row_groups, 1); + EXPECT_EQ(enabled.filtered_row_groups_by_min_max, 1); + EXPECT_EQ(disabled.materialize_calls, 0); + EXPECT_EQ(disabled.prepare_literals_calls, 0); + EXPECT_EQ(disabled.filtered_row_groups, 0); + EXPECT_EQ(disabled.filtered_row_groups_by_min_max, 0); + ASSERT_EQ(enabled.ids, disabled.ids); + ASSERT_EQ(enabled.ids.size(), 200); + EXPECT_EQ(enabled.ids.front(), 1000); + EXPECT_EQ(enabled.ids.back(), 1199); + } +} + +TEST_F(NewOrcReaderTest, SargDirectInNullSafeOrFallsBackBeforeLiteralConversion) { + const auto multi_stripe_file_path = (_test_dir / "sarg_direct_in_null_safe_or.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + constexpr int32_t in_list_size = 1024; + DirectInScanResult enabled; + DirectInScanResult disabled; + ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size, true, in_list_size, + DirectInPredicateShape::OR_WITH_NULL_SAFE_EQUAL, &enabled) + .ok()); + ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size, false, in_list_size, + DirectInPredicateShape::OR_WITH_NULL_SAFE_EQUAL, &disabled) + .ok()); + + EXPECT_EQ(enabled.materialize_calls, 1); + EXPECT_EQ(enabled.prepare_literals_calls, 0); + EXPECT_EQ(enabled.filtered_row_groups, 0); + EXPECT_EQ(enabled.filtered_row_groups_by_min_max, 0); + EXPECT_EQ(disabled.materialize_calls, 0); + EXPECT_EQ(disabled.prepare_literals_calls, 0); + EXPECT_EQ(disabled.filtered_row_groups, 0); + EXPECT_EQ(disabled.filtered_row_groups_by_min_max, 0); + ASSERT_EQ(enabled.ids, disabled.ids); + ASSERT_EQ(enabled.ids.size(), 200); + EXPECT_EQ(enabled.ids.front(), 1000); + EXPECT_EQ(enabled.ids.back(), 1199); +} + +TEST_F(NewOrcReaderTest, SargDirectInOverLimitFallsBackBeforeMaterialization) { + const auto multi_stripe_file_path = (_test_dir / "sarg_direct_in_over_limit.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + constexpr int32_t max_pushdown_conditions_per_column = 1024; + constexpr int32_t in_list_size = max_pushdown_conditions_per_column + 1; + DirectInScanResult enabled; + DirectInScanResult disabled; + ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size, true, + max_pushdown_conditions_per_column, + DirectInPredicateShape::ROOT, &enabled) + .ok()); + ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size, false, + max_pushdown_conditions_per_column, + DirectInPredicateShape::ROOT, &disabled) + .ok()); + + EXPECT_EQ(enabled.materialize_calls, 0); + EXPECT_EQ(enabled.prepare_literals_calls, 0); + EXPECT_EQ(enabled.filtered_row_groups, 0); + EXPECT_EQ(enabled.filtered_row_groups_by_min_max, 0); + EXPECT_EQ(disabled.materialize_calls, 0); + EXPECT_EQ(disabled.prepare_literals_calls, 0); + EXPECT_EQ(disabled.filtered_row_groups, 0); + EXPECT_EQ(disabled.filtered_row_groups_by_min_max, 0); + ASSERT_EQ(enabled.ids, disabled.ids); + ASSERT_EQ(enabled.ids.size(), 200); + EXPECT_EQ(enabled.ids.front(), 1000); + EXPECT_EQ(enabled.ids.back(), 1199); +} + TEST_F(NewOrcReaderTest, SargNullAwareRuntimeFilterDoesNotPruneNullStripe) { const auto multi_stripe_file_path = (_test_dir / "sarg_null_aware_runtime_filter.orc").string(); write_two_stripe_orc_nullable_int_file(multi_stripe_file_path); @@ -6936,6 +7399,113 @@ TEST_F(NewOrcReaderTest, SargSlotToSlotNullSafeEqualDoesNotPruneStripes) { EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); } +TEST_F(NewOrcReaderTest, SargPartialAndRespectsNegationContext) { + const auto multi_stripe_file_path = (_test_dir / "sarg_partial_and_not.orc").string(); + write_multi_stripe_orc_int_file(multi_stripe_file_path); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto partial_and = []() -> VExprSPtr { + return std::make_shared( + TExprOpcode::COMPOUND_AND, + VExprSPtrs {std::make_shared(0, 500), + std::make_shared(0, 1100.5F)}); + }; + auto verify = [&](const auto& conjunct_factory, const std::vector& expected_ids, + int64_t expected_filtered_row_groups) { + std::array, 2> result_ids; + for (size_t run = 0; run < result_ids.size(); ++run) { + const bool enable_filter_by_min_max = run == 0; + auto reader = create_reader_for_path(multi_stripe_file_path); + TQueryOptions query_options; + query_options.__set_enable_orc_filter_by_min_max(enable_filter_by_min_max); + RuntimeState state {query_options, TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared(conjunct_factory())); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& ids_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& ids = assert_cast(ids_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + ASSERT_FALSE(ids_nullable.is_null_at(row)); + result_ids[run].push_back(ids.get_element(row)); + } + } + + const auto filtered_row_groups = + enable_filter_by_min_max ? expected_filtered_row_groups : 0; + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, filtered_row_groups); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, + filtered_row_groups); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, filtered_row_groups * 200); + } + + EXPECT_EQ(result_ids[0], expected_ids); + EXPECT_EQ(result_ids[0], result_ids[1]); + }; + + std::vector expected_not_ids; + for (int32_t id = 1; id <= 200; ++id) { + expected_not_ids.push_back(id); + } + for (int32_t id = 1000; id <= 1100; ++id) { + expected_not_ids.push_back(id); + } + verify( + [&]() -> VExprSPtr { + return std::make_shared(TExprOpcode::COMPOUND_NOT, + VExprSPtrs {partial_and()}); + }, + expected_not_ids, 0); + + std::vector expected_and_ids; + for (int32_t id = 1101; id <= 1199; ++id) { + expected_and_ids.push_back(id); + } + verify(partial_and, expected_and_ids, 1); + verify( + [&]() -> VExprSPtr { + auto inner_not = std::make_shared( + TExprOpcode::COMPOUND_NOT, VExprSPtrs {partial_and()}); + return std::make_shared(TExprOpcode::COMPOUND_NOT, + VExprSPtrs {std::move(inner_not)}); + }, + expected_and_ids, 1); + + std::vector expected_nested_ids; + expected_nested_ids.insert(expected_nested_ids.end(), expected_not_ids.begin(), + expected_not_ids.begin() + 200); + expected_nested_ids.insert(expected_nested_ids.end(), expected_and_ids.begin(), + expected_and_ids.end()); + verify( + [&]() -> VExprSPtr { + auto inner_not = std::make_shared( + TExprOpcode::COMPOUND_NOT, VExprSPtrs {partial_and()}); + auto outer_and = std::make_shared( + TExprOpcode::COMPOUND_AND, + VExprSPtrs {std::move(inner_not), + std::make_shared(0, 500)}); + return std::make_shared(TExprOpcode::COMPOUND_NOT, + VExprSPtrs {std::move(outer_and)}); + }, + expected_nested_ids, 0); +} + TEST_F(NewOrcReaderTest, SargNullSafeEqualInsideOrDoesNotPruneStripes) { const auto multi_stripe_file_path = (_test_dir / "sarg_null_safe_equal_inside_or.orc").string(); write_multi_stripe_orc_int_file(multi_stripe_file_path); @@ -7155,6 +7725,138 @@ TEST_F(NewOrcReaderTest, SargBinaryVarbinaryLiteralConjunctPrunesRowGroups) { EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); } +TEST_F(NewOrcReaderTest, SargExceptionsFallBackToResidualScan) { + const auto valid_file_path = (_test_dir / "sarg_build_exception.orc").string(); + const auto corrupt_statistics_file_path = (_test_dir / "sarg_corrupt_statistics.orc").string(); + write_multi_stripe_orc_binary_file(valid_file_path); + write_multi_stripe_orc_binary_file(corrupt_statistics_file_path); + const auto valid_stripe_layout = get_orc_stripe_layout(valid_file_path); + const auto corrupt_statistics_stripe_layout = + get_orc_stripe_layout(corrupt_statistics_file_path); + ASSERT_EQ(valid_stripe_layout.size(), 2); + ASSERT_EQ(corrupt_statistics_stripe_layout.size(), 2); + ASSERT_TRUE(corrupt_orc_stripe_statistics(corrupt_statistics_file_path)); + + auto scan_first_stripe = [&](const std::string& file_path, const OrcStripeLayout& stripe, + bool enable_filter_by_min_max, bool add_real_residual_filter, + std::vector* result_values, Status* open_status) { + auto reader = create_reader_with_range(file_path, static_cast(stripe.offset), + static_cast(stripe.length)); + TQueryOptions query_options; + query_options.__set_enable_orc_filter_by_min_max(enable_filter_by_min_max); + RuntimeState state {query_options, TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(VExprContext::create_shared( + std::make_shared( + 0, std::make_shared(), "zzz_1000", "value"))); + if (add_real_residual_filter) { + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared( + 0, std::make_shared(), "aaa_1050", "value"))); + } + *open_status = reader->open(request); + if (!open_status->ok()) { + return; + } + + bool eof = false; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (eof || rows == 0) { + continue; + } + const auto& values_nullable = + assert_cast(*block.get_by_position(0).column); + const auto& values = + assert_cast(values_nullable.get_nested_column()); + for (size_t row = 0; row < rows; ++row) { + ASSERT_FALSE(values_nullable.is_null_at(row)); + result_values->push_back(values.get_data_at(row).to_string()); + } + } + + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); + ASSERT_TRUE(reader->close().ok()); + }; + + std::vector without_sarg; + Status without_sarg_open_status; + scan_first_stripe(corrupt_statistics_file_path, corrupt_statistics_stripe_layout[0], false, + false, &without_sarg, &without_sarg_open_status); + ASSERT_TRUE(without_sarg_open_status.ok()) << without_sarg_open_status; + ASSERT_EQ(without_sarg.size(), 200); + EXPECT_EQ(without_sarg.front(), "aaa_1000"); + EXPECT_EQ(without_sarg.back(), "aaa_1199"); + + int injection_count = 0; + std::vector after_build_failure; + Status build_failure_open_status; + { + ScopedDebugPoint debug_point( + "OrcReader._init_search_argument_from_local_filters.inject_failure", [&]() { + ++injection_count; + throw std::runtime_error("injected ORC SARG build failure"); + }); + scan_first_stripe(valid_file_path, valid_stripe_layout[0], true, false, + &after_build_failure, &build_failure_open_status); + } + EXPECT_EQ(injection_count, 1); + ASSERT_TRUE(build_failure_open_status.ok()) << build_failure_open_status; + EXPECT_EQ(after_build_failure, without_sarg); + + std::vector after_evaluation_failure; + Status evaluation_failure_open_status; + scan_first_stripe(corrupt_statistics_file_path, corrupt_statistics_stripe_layout[0], true, + false, &after_evaluation_failure, &evaluation_failure_open_status); + ASSERT_TRUE(evaluation_failure_open_status.ok()) << evaluation_failure_open_status; + EXPECT_EQ(after_evaluation_failure, without_sarg); + + std::vector residual_without_sarg; + Status residual_without_sarg_open_status; + scan_first_stripe(corrupt_statistics_file_path, corrupt_statistics_stripe_layout[0], false, + true, &residual_without_sarg, &residual_without_sarg_open_status); + ASSERT_TRUE(residual_without_sarg_open_status.ok()) << residual_without_sarg_open_status; + const std::vector expected_residual_values = {"aaa_1050"}; + EXPECT_EQ(residual_without_sarg, expected_residual_values); + + std::vector residual_after_evaluation_failure; + Status residual_evaluation_failure_open_status; + scan_first_stripe(corrupt_statistics_file_path, corrupt_statistics_stripe_layout[0], true, true, + &residual_after_evaluation_failure, &residual_evaluation_failure_open_status); + ASSERT_TRUE(residual_evaluation_failure_open_status.ok()) + << residual_evaluation_failure_open_status; + EXPECT_EQ(residual_after_evaluation_failure, expected_residual_values); + + int memory_failure_injection_count = 0; + std::vector after_memory_failure; + Status memory_failure_open_status; + { + ScopedDebugPoint debug_point( + "OrcReader._init_search_argument_from_local_filters.inject_failure", [&]() { + ++memory_failure_injection_count; + throw Exception(ErrorCode::MEM_ALLOC_FAILED, + "injected ORC SARG allocation failure"); + }); + scan_first_stripe(valid_file_path, valid_stripe_layout[0], true, false, + &after_memory_failure, &memory_failure_open_status); + } + EXPECT_EQ(memory_failure_injection_count, 1); + EXPECT_TRUE(memory_failure_open_status.is()) + << memory_failure_open_status; + EXPECT_TRUE(after_memory_failure.empty()); +} + TEST_F(NewOrcReaderTest, SargBinaryCastFromVarbinaryToStringConjunctPrunesRowGroups) { const auto multi_stripe_file_path = (_test_dir / "binary_varbinary_to_string_cast_sarg.orc").string(); diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp new file mode 100644 index 00000000000000..599c930f312856 --- /dev/null +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -0,0 +1,4146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/custom_allocator.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/column_reader.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader.h" +#include "util/block_compression.h" +#include "util/coding.h" +#include "util/faststring.h" +#include "util/rle_encoding.h" +#include "util/thrift_util.h" +#include "util/timezone_utils.h" + +namespace doris::format::parquet::native { +namespace { + +VExprSPtr create_int32_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t literal, + bool literal_on_left = false) { + const auto int_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + auto slot = VSlotRef::create_shared(column_id, column_id, -1, int_type, "plain_int"); + auto value = VLiteral::create_shared(int_type, Field::create_field(literal)); + if (literal_on_left) { + root->add_child(value); + root->add_child(slot); + } else { + root->add_child(slot); + root->add_child(value); + } + return root; +} + +VExprSPtr create_float_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, float literal) { + const auto float_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({float_type->to_thrift(), float_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + root->add_child(VSlotRef::create_shared(column_id, column_id, -1, float_type, "plain_float")); + root->add_child(VLiteral::create_shared(float_type, Field::create_field(literal))); + return root; +} + +TEST(ParquetV2NativeDecoderTest, RawExprEvaluatesPhysicalValuesWithoutColumn) { + const std::array values = {1, 4, 7, 10, 13, 16}; + std::array matches {}; + matches.fill(1); + const auto greater_equal = create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 7); + const auto less_than = create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 16); + const auto int_type = std::make_shared(); + + ASSERT_TRUE(greater_equal->can_execute_on_raw_fixed_values(int_type, 0)); + ASSERT_TRUE(greater_equal + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + ASSERT_TRUE(less_than + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1, 1, 0})); + + matches.fill(1); + const auto literal_less_than_slot = + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 7, true); + ASSERT_TRUE(literal_less_than_slot + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, RawExprPreservesFloatNanOrdering) { + const float nan = std::numeric_limits::quiet_NaN(); + const std::array values {-1, 0, 1, nan}; + const auto float_type = std::make_shared(); + + std::array matches {}; + matches.fill(1); + const auto greater_than_zero = create_float_raw_comparison(0, "gt", TExprOpcode::GT, 0); + ASSERT_TRUE(greater_than_zero + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1})); + + matches.fill(1); + const auto equals_nan = create_float_raw_comparison(0, "eq", TExprOpcode::EQ, nan); + ASSERT_TRUE(equals_nan + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1})); +} + +TEST(ParquetV2NativeDecoderTest, RawFixedFilterSupportsIdentityWidthEncodingTypes) { + using Reader = ColumnChunkReader; + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::INT32)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::INT64)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::FLOAT)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::BYTE_STREAM_SPLIT, + tparquet::Type::DOUBLE)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::DELTA_BINARY_PACKED, + tparquet::Type::INT32)); + EXPECT_TRUE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::DELTA_BINARY_PACKED, + tparquet::Type::INT64)); + EXPECT_FALSE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::DELTA_BINARY_PACKED, + tparquet::Type::FLOAT)); + EXPECT_FALSE(Reader::supports_raw_fixed_filter_encoding(tparquet::Encoding::RLE_DICTIONARY, + tparquet::Type::INT32)); +} + +class RejectFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return Status::InternalError("Unexpected fixed dictionary"); + } +}; + +class CaptureBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + refs.assign(values, values + num_values); + return Status::OK(); + } + + std::vector refs; +}; + +class CapturePlainBinaryLayoutConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + legacy_consume_called = true; + return Status::InternalError("PLAIN BYTE_ARRAY used the legacy StringRef path"); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + base = encoded_data; + source_offsets.assign(payload_offsets, payload_offsets + num_values); + output_offsets.assign(value_offsets, value_offsets + num_values + 1); + spans = value_spans; + return Status::OK(); + } + + const char* base = nullptr; + bool legacy_consume_called = false; + std::vector source_offsets; + std::vector output_offsets; + std::vector spans; +}; + +class CaptureFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + ++consume_calls; + if (width == 0) { + width = value_width; + } + DORIS_CHECK_EQ(width, value_width); + bytes.insert(bytes.end(), values, values + num_values * value_width); + return Status::OK(); + } + + template + std::vector values() const { + DORIS_CHECK_EQ(width, sizeof(T)); + DORIS_CHECK_EQ(bytes.size() % sizeof(T), 0); + std::vector result(bytes.size() / sizeof(T)); + memcpy(result.data(), bytes.data(), bytes.size()); + return result; + } + + size_t width = 0; + size_t consume_calls = 0; + std::vector bytes; +}; + +TEST(ParquetV2NativeDecoderTest, FragmentedPlainSelectionUsesOneConsumerBatch) { + std::array input {}; + std::iota(input.begin(), input.end(), 0); + Slice data(reinterpret_cast(input.data()), input.size() * sizeof(int32_t)); + FixLengthPlainDecoder decoder; + decoder.set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder.set_data(&data).ok()); + + ParquetSelection selection { + .total_values = input.size(), .selected_values = input.size() / 2, .ranges = {}}; + for (size_t row = 0; row < input.size(); row += 2) { + selection.ranges.push_back({.first = row, .count = 1}); + } + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder.decode_selected_fixed_values(selection, consumer).ok()); + + EXPECT_EQ(consumer.consume_calls, 1); + std::vector expected; + for (int32_t value = 0; value < static_cast(input.size()); value += 2) { + expected.push_back(value); + } + EXPECT_EQ(consumer.values(), expected); +} + +class ScriptedDictionaryMaterializationSource final : public ParquetDecodeSource { +public: + ScriptedDictionaryMaterializationSource(std::vector ids, bool prefer_indices) + : _ids(std::move(ids)), _prefer_indices(prefer_indices) {} + + Status decode_fixed_values(size_t, ParquetFixedValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no fixed values"); + } + + Status decode_binary_values(size_t, ParquetBinaryValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no binary values"); + } + + Status skip_values(size_t) override { return Status::OK(); } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++index_batches; + *indices = _ids; + return Status::OK(); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++direct_batches; + return consumer.consume_indices(_ids.data(), _ids.size()); + } + + bool prefer_dictionary_index_materialization(size_t) const override { return _prefer_indices; } + + size_t direct_batches = 0; + size_t index_batches = 0; + +private: + std::vector _ids; + bool _prefer_indices; +}; + +const RowRanges& scripted_row_ranges() { + static const RowRanges ranges; + return ranges; +} + +class ScriptedColumnReader final : public ColumnReader { +public: + ScriptedColumnReader(size_t rows, size_t values, bool eof, std::vector rep_levels, + std::vector def_levels) + : ColumnReader(scripted_row_ranges(), rows, nullptr, nullptr), + _rows(rows), + _values(values), + _eof(eof), + _rep_levels(std::move(rep_levels)), + _def_levels(std::move(def_levels)) {} + + Status read_column_data(ColumnPtr& column, const DataTypePtr&, + const std::shared_ptr&, FilterMap&, size_t, + size_t* read_rows, bool* eof, bool, int64_t = -1) override { + if (_used) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + column = IColumn::mutate(std::move(column)); + column->assert_mutable()->insert_many_defaults(_values); + *read_rows = _rows; + *eof = _eof; + _used = true; + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t* read_rows, bool* eof) override { + *read_rows = _rows; + *eof = _eof; + return Status::OK(); + } + + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { return {}; } + void close() override {} + void release_batch_scratch(size_t) override {} + void reset_filter_map_index() override {} + +private: + size_t _rows; + size_t _values; + bool _eof; + bool _used = false; + std::vector _rep_levels; + std::vector _def_levels; +}; + +class MemoryBufferedReader final : public io::BufferedStreamReader { +public: + explicit MemoryBufferedReader(std::vector data) : _data(std::move(data)) {} + + Status read_bytes(const uint8_t** buf, uint64_t offset, size_t bytes_to_read, + const io::IOContext*) override { + if (offset > _data.size() || bytes_to_read > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + *buf = _data.data() + offset; + return Status::OK(); + } + Status read_bytes(Slice& slice, uint64_t offset, const io::IOContext*) override { + if (offset > _data.size() || slice.size > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + slice.data = reinterpret_cast(_data.data() + offset); + return Status::OK(); + } + std::string path() override { return "memory.parquet"; } + int64_t mtime() const override { return 0; } + +private: + std::vector _data; +}; + +class NativeDecoderMemoryFileReader final : public io::FileReader { +public: + explicit NativeDecoderMemoryFileReader(std::vector data) + : _data(std::move(data)), _path("native-decoder-memory.parquet") {} + + Status close() override { + _closed = true; + return Status::OK(); + } + const io::Path& path() const override { return _path; } + size_t size() const override { return _data.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 1; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext*) override { + if (offset > _data.size() || result.size > _data.size() - offset) { + return Status::IOError("native decoder memory read exceeds file"); + } + memcpy(result.data, _data.data() + offset, result.size); + *bytes_read = result.size; + return Status::OK(); + } + +private: + std::vector _data; + io::Path _path; + bool _closed = false; +}; + +std::shared_ptr<::parquet::ColumnDescriptor> descriptor(::parquet::Type::type physical_type) { + auto node = ::parquet::schema::PrimitiveNode::Make("value", ::parquet::Repetition::REQUIRED, + physical_type); + return std::make_shared<::parquet::ColumnDescriptor>(node, 0, 0); +} + +DorisUniqueBufferPtr make_byte_array_dictionary(const std::vector& values, + int32_t* length) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + *length = static_cast(total_size); + auto dictionary = make_unique_buffer(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(dictionary.get() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(dictionary.get() + offset, value.data(), value.size()); + offset += value.size(); + } + return dictionary; +} + +std::vector encode_plain_byte_arrays(const std::vector& values) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + std::vector encoded(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(encoded.data() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(encoded.data() + offset, value.data(), value.size()); + offset += value.size(); + } + return encoded; +} + +std::vector serialize_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +Status materialize_level_only_page(bool data_page_v2, tparquet::Type::type physical_type, + tparquet::Encoding::type encoding, bool all_null) { + constexpr size_t VALUE_COUNT = 3; + const std::vector encoded_levels {static_cast(VALUE_COUNT << 1), + static_cast(all_null ? 0 : 1)}; + std::vector payload; + tparquet::PageHeader header; + if (data_page_v2) { + payload = encoded_levels; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(VALUE_COUNT); + header.data_page_header_v2.__set_num_nulls(all_null ? VALUE_COUNT : 0); + header.data_page_header_v2.__set_num_rows(VALUE_COUNT); + header.data_page_header_v2.__set_encoding(encoding); + header.data_page_header_v2.__set_definition_levels_byte_length(encoded_levels.size()); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } else { + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), encoded_levels.size()); + payload.insert(payload.end(), encoded_levels.begin(), encoded_levels.end()); + header.type = tparquet::PageType::DATA_PAGE; + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(VALUE_COUNT); + header.data_page_header.__set_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + } + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(VALUE_COUNT); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = physical_type; + if (physical_type == tparquet::Type::BOOLEAN) { + field.data_type = std::make_shared(); + } else { + field.data_type = std::make_shared(); + } + field.definition_level = 1; + field.parquet_schema.__set_type(physical_type); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, VALUE_COUNT, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + const auto load_status = chunk_reader.load_page_data(); + EXPECT_TRUE(load_status.ok()) << load_status; + RETURN_IF_ERROR(load_status); + + level_t level = -1; + EXPECT_EQ(chunk_reader.def_level_decoder().get_next_run(&level, VALUE_COUNT), VALUE_COUNT); + EXPECT_EQ(level, all_null ? 0 : 1); + FilterMap filter; + RETURN_IF_ERROR(filter.init(nullptr, VALUE_COUNT, false)); + NullMap null_map; + ColumnSelectVector select_vector; + const std::vector null_runs {static_cast(all_null ? 0 : VALUE_COUNT), + static_cast(all_null ? VALUE_COUNT : 0)}; + RETURN_IF_ERROR(select_vector.init(null_runs, VALUE_COUNT, &null_map, &filter, 0)); + auto column = field.data_type->create_column(); + ParquetDecodeContext decode_context; + static const auto utc = cctz::utc_time_zone(); + RETURN_IF_ERROR(init_decode_context_for_test(field, &utc, &decode_context)); + ParquetMaterializationState state; + state.enable_strict_mode = true; + const auto status = chunk_reader.materialize_values(column, *field.data_type->get_serde(), + decode_context, state, select_vector); + if (status.ok()) { + EXPECT_EQ(column->size(), VALUE_COUNT); + EXPECT_EQ(null_map, NullMap(VALUE_COUNT, all_null ? 1 : 0)); + } + return status; +} + +Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, + tparquet::CompressionCodec::type codec, bool preload_page_cache = false, + tparquet::Type::type physical_type = tparquet::Type::INT32) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + RETURN_IF_ERROR(serializer.serialize(&header, &bytes)); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + const size_t chunk_size = bytes.size(); + const std::string page_cache_file_key = "native-scripted-page-" + + std::to_string(static_cast(header.type)) + "-" + + std::to_string(header.compressed_page_size) + "-" + + std::to_string(header.uncompressed_page_size); + if (preload_page_cache) { + auto* page = new DataPage(bytes.size(), true, segment_v2::DATA_PAGE); + memcpy(page->data(), bytes.data(), bytes.size()); + page->reset_size(bytes.size()); + PageCacheHandle handle; + StoragePageCache::instance()->insert( + StoragePageCache::CacheKey(page_cache_file_key, chunk_size, 0), page, &handle, + segment_v2::DATA_PAGE); + } + MemoryBufferedReader reader(std::move(bytes)); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(physical_type); + chunk.meta_data.__set_codec(codec); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(chunk_size); + if (header.type == tparquet::PageType::DICTIONARY_PAGE) { + chunk.meta_data.__set_dictionary_page_offset(0); + chunk.meta_data.__set_data_page_offset(0); + } else { + chunk.meta_data.__set_data_page_offset(0); + } + NativeFieldSchema field; + field.physical_type = physical_type; + if (physical_type == tparquet::Type::BYTE_ARRAY) { + field.data_type = std::make_shared(); + } else { + field.data_type = std::make_shared(); + } + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(preload_page_cache, page_cache_file_key); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + return chunk_reader.load_page_data(); +} + +Status load_malformed_nested_page(tparquet::PageHeader header, const std::vector& payload, + int64_t metadata_values = std::numeric_limits::max(), + level_t max_repetition_level = 1) { + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(metadata_values); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = max_repetition_level; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + std::vector rep_levels; + size_t result_rows = 0; + bool cross_page = false; + return chunk_reader.load_page_nested_rows(rep_levels, 1, &result_rows, &cross_page); +} + +Status materialize_plain_int96(const std::vector& values, + const std::vector& filter_values = {}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__set_uncompressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values.size()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto bytes = serialize_page( + header, std::vector(reinterpret_cast(values.data()), + reinterpret_cast(values.data()) + + values.size() * sizeof(ParquetInt96Timestamp))); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT96); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(values.size()); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT96; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, values.size(), + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::INT96; + static const auto utc = cctz::utc_time_zone(); + decode_context.timezone = &utc; + ParquetMaterializationState state; + state.enable_strict_mode = true; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.empty() ? nullptr : filter_values.data(), + filter_values.size(), false)); + ColumnSelectVector select_vector; + const std::vector run_length_null_map {static_cast(values.size()), 0}; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, values.size(), nullptr, &filter, 0)); + return chunk_reader.materialize_values(column, *type.get_serde(), decode_context, state, + select_vector); +} + +template +Status materialize_selected_plain_fixed( + const std::vector& physical_values, size_t logical_values, + const std::vector& run_length_null_map, const std::vector& filter_values, + tparquet::Type::type parquet_physical_type, ParquetPhysicalType decode_physical_type, + const DataType& type, MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics, bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(PhysicalType)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(PhysicalType)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(static_cast(logical_values)); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(PhysicalType)); + if (!payload.empty()) { + memcpy(payload.data(), physical_values.data(), payload.size()); + } + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(parquet_physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = parquet_physical_type; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = decode_physical_type; + if (context_override != nullptr) { + decode_context = *context_override; + } + ParquetMaterializationState state; + state.enable_strict_mode = strict_mode; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +template +Status materialize_selected_plain_int32(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT32, + ParquetPhysicalType::INT32, type, column, null_map, + statistics, strict_mode, context_override); +} + +template +Status materialize_selected_plain_int64(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + const ParquetDecodeContext& context) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT64, + ParquetPhysicalType::INT64, type, column, null_map, + statistics, false, &context); +} + +template +Status materialize_selected_dictionary_fixed( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + tparquet::Type::type parquet_physical_type, ParquetPhysicalType decode_physical_type, + const ParquetDecodeContext* context_override = nullptr) { + std::vector dictionary_payload(dictionary.size() * sizeof(PhysicalType)); + memcpy(dictionary_payload.data(), dictionary.data(), dictionary_payload.size()); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + // Real Parquet files place column chunks after the file header. Keep the dictionary offset + // non-zero because zero is the metadata sentinel for "no dictionary page". + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 4); + for (const auto id : physical_ids) { + encoder.Put(id); + } + // Parquet bit-packed dictionary runs declare groups of eight; emit the trailing padding so a + // decoder that buffers the declared run never reads beyond the synthetic page. + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 4; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(parquet_physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = parquet_physical_type; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = decode_physical_type; + if (context_override != nullptr) { + decode_context = *context_override; + } + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +template +Status materialize_selected_dictionary_int32( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics) { + return materialize_selected_dictionary_fixed( + dictionary, physical_ids, logical_values, run_length_null_map, filter_values, type, + column, null_map, statistics, tparquet::Type::INT32, ParquetPhysicalType::INT32); +} + +template +Status materialize_selected_dictionary_int64( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + const ParquetDecodeContext& context) { + return materialize_selected_dictionary_fixed(dictionary, physical_ids, logical_values, + run_length_null_map, filter_values, type, column, + null_map, statistics, tparquet::Type::INT64, + ParquetPhysicalType::INT64, &context); +} + +Status materialize_selected_dictionary_strings(const std::vector& dictionary, + const std::vector& physical_ids, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics) { + auto dictionary_payload = encode_plain_byte_arrays(dictionary); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (const auto id : physical_ids) { + encoder.Put(id); + } + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 2; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::BYTE_ARRAY); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = tparquet::Type::BYTE_ARRAY; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeString type; + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::BYTE_ARRAY; + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +TEST(ParquetV2NativeDecoderTest, RawExprMapsNullableSparseRowsDirectly) { + const std::vector physical_values {1, 4, 7, 10, 13}; + constexpr size_t LOGICAL_VALUES = 7; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(LOGICAL_VALUES); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(int32_t)); + memcpy(payload.data(), physical_values.data(), payload.size()); + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(LOGICAL_VALUES); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, LOGICAL_VALUES, + nullptr, page_context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + + const std::vector null_runs {1, 1, 2, 1, 2}; + const std::vector input_filter {1, 0, 1, 1, 1, 0, 1}; + FilterMap filter; + ASSERT_TRUE(filter.init(input_filter.data(), input_filter.size(), false).ok()); + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector.init(null_runs, LOGICAL_VALUES, nullptr, &filter, 0).ok()); + const VExprSPtrs predicates {create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 4), + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 13)}; + NullMap selected_nulls; + IColumn::Filter physical_matches; + auto projected_column = make_nullable(field.data_type)->create_column(); + IColumn::Filter row_filter; + bool used_filter = false; + ASSERT_TRUE(chunk_reader + .filter_fixed_width_values(predicates, 0, select_vector, &selected_nulls, + &physical_matches, projected_column.get(), + &row_filter, &used_filter) + .ok()); + + EXPECT_TRUE(used_filter); + EXPECT_EQ(selected_nulls, (NullMap {0, 0, 0, 1, 0})); + EXPECT_EQ(physical_matches, (IColumn::Filter {0, 1, 1, 0})); + EXPECT_EQ(row_filter, (IColumn::Filter {0, 1, 1, 0, 0})); + ASSERT_EQ(projected_column->size(), 2); + EXPECT_FALSE(projected_column->is_null_at(0)); + EXPECT_FALSE(projected_column->is_null_at(1)); + const auto& projected_values = + assert_cast( + assert_cast(*projected_column).get_nested_column()) + .get_data(); + EXPECT_EQ(projected_values[0], 4); + EXPECT_EQ(projected_values[1], 7); + EXPECT_EQ(chunk_reader.remaining_num_values(), 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainPrimitiveSelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 0, 500, 0, 1000, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDecimalSelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeDecimal32 type(9, 0); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 9, + .decimal_scale = 0}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + const auto& values = assert_cast(*column).get_data(); + ASSERT_EQ(values.size(), 5); + EXPECT_EQ(values[0].value, 0); + EXPECT_EQ(values[1].value, 500); + EXPECT_EQ(values[2].value, 0); + EXPECT_EQ(values[3].value, 1000); + EXPECT_EQ(values[4].value, 0); + EXPECT_EQ(null_map, (NullMap {0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateSelectionBatchesPhysicalPayload) { + const std::vector physical_days {0, 1, 2, 3, 4}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateV2 type; + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_days, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-04"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateTimeSelectionBatchesPhysicalPayload) { + const std::vector physical_micros {0, 1'000'000, 2'000'000, 3'000'000, 4'000'000}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS}; + ASSERT_TRUE(materialize_selected_plain_int64(physical_micros, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000000"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-01 00:00:03.000000"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NegativeNanosFloorAcrossPlainAndDictionaryTimestamps) { + const std::vector nanos {-1, -999, -1000, -1001, 0, 1001}; + const std::vector dictionary_ids {0, 1, 2, 3, 4, 5}; + const std::vector no_nulls {static_cast(nanos.size()), 0}; + const std::vector select_all(nanos.size(), 1); + const std::vector expected { + "1969-12-31 23:59:59.999999", "1969-12-31 23:59:59.999999", + "1969-12-31 23:59:59.999999", "1969-12-31 23:59:59.999998", + "1970-01-01 00:00:00.000000", "1970-01-01 00:00:00.000001"}; + const ParquetDecodeContext local_context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::NANOS}; + const ParquetDecodeContext utc_context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::NANOS, + .timestamp_is_adjusted_to_utc = true}; + + for (const bool dictionary : {false, true}) { + SCOPED_TRACE(dictionary ? "dictionary" : "plain"); + DataTypeDateTimeV2 datetime_type(6); + auto datetime_column = datetime_type.create_column(); + NullMap datetime_nulls; + ColumnChunkReaderStatistics datetime_statistics; + const auto datetime_status = + dictionary ? materialize_selected_dictionary_int64( + nanos, dictionary_ids, nanos.size(), no_nulls, select_all, + datetime_type, &datetime_column, &datetime_nulls, + &datetime_statistics, local_context) + : materialize_selected_plain_int64(nanos, nanos.size(), no_nulls, + select_all, datetime_type, + &datetime_column, &datetime_nulls, + &datetime_statistics, local_context); + ASSERT_TRUE(datetime_status.ok()) << datetime_status; + ASSERT_EQ(datetime_column->size(), expected.size()); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(datetime_type.to_string(*datetime_column, row), expected[row]); + } + + DataTypeTimeStampTz timestamptz_type(6); + auto timestamptz_column = timestamptz_type.create_column(); + NullMap timestamptz_nulls; + ColumnChunkReaderStatistics timestamptz_statistics; + const auto timestamptz_status = + dictionary ? materialize_selected_dictionary_int64( + nanos, dictionary_ids, nanos.size(), no_nulls, select_all, + timestamptz_type, ×tamptz_column, ×tamptz_nulls, + ×tamptz_statistics, utc_context) + : materialize_selected_plain_int64( + nanos, nanos.size(), no_nulls, select_all, timestamptz_type, + ×tamptz_column, ×tamptz_nulls, + ×tamptz_statistics, utc_context); + ASSERT_TRUE(timestamptz_status.ok()) << timestamptz_status; + const auto& tz_column = assert_cast(*timestamptz_column); + const auto utc = cctz::utc_time_zone(); + for (size_t row = 0; row < expected.size(); ++row) { + EXPECT_EQ(tz_column.get_element(row).to_string(utc, 6), expected[row] + "+00:00"); + } + } +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseDictionarySelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector dictionary(16); + std::iota(dictionary.begin(), dictionary.end(), 100); + // One value followed by nineteen NULLs exercises the sparse-null threshold (<10%). + std::vector null_runs; + for (size_t row = 0; row < LOGICAL_VALUES;) { + null_runs.push_back(1); + ++row; + const auto null_count = std::min(19, LOGICAL_VALUES - row); + null_runs.push_back(cast_set(null_count)); + row += null_count; + } + std::vector physical_ids((LOGICAL_VALUES + 19) / 20); + for (size_t index = 0; index < physical_ids.size(); ++index) { + physical_ids[index] = static_cast(index % dictionary.size()); + } + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + const auto status = materialize_selected_dictionary_int32( + dictionary, physical_ids, LOGICAL_VALUES, null_runs, filter, type, &column, &null_map, + &statistics); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 100, 102, 0, 104, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationSkipsFailureScanWhenDictionaryIsClean) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + auto& dictionary = assert_cast(*state.typed_dictionary).get_data(); + dictionary = {10, 20, 30, 40}; + state.dictionary_indices = {3, 0, 2, 1, 3}; + state.dictionary_conversion_failures.resize_fill(dictionary.size(), 0); + + auto output = ColumnInt32::create(); + ASSERT_TRUE(state.materialize_dictionary(*output).ok()); + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_failure_scan_rows, 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationUsesCacheAwareExecutionShape) { + auto verify_strategy = [](bool prefer_indices, + ParquetDictionaryMaterializationStrategy expected_strategy, + size_t expected_direct_batches, size_t expected_index_batches) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + assert_cast(*state.typed_dictionary).get_data() = {10, 20, 30, 40}; + ScriptedDictionaryMaterializationSource source({3, 0, 2, 1, 3}, prefer_indices); + auto output = ColumnInt32::create(); + + const auto status = state.materialize_dictionary(*output, source, 5); + EXPECT_TRUE(status.ok()) << status; + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_materialization_strategy, expected_strategy); + EXPECT_EQ(source.direct_batches, expected_direct_batches); + EXPECT_EQ(source.index_batches, expected_index_batches); + }; + + verify_strategy(false, ParquetDictionaryMaterializationStrategy::DIRECT, 1, 0); + verify_strategy(true, ParquetDictionaryMaterializationStrategy::INDICES, 0, 1); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryProbeMaterializesTypedValuesOnlyOnce) { + const std::array dictionary {10, 20}; + std::vector dictionary_payload(sizeof(dictionary)); + memcpy(dictionary_payload.data(), dictionary.data(), dictionary_payload.size()); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values(dictionary.size()); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + std::vector bytes(1, 0); + const auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 1); + for (const uint32_t id : {1U, 0U, 0U, 0U, 0U, 0U, 0U, 0U}) { + encoder.Put(id); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 1; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(2); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(data_page_offset); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + auto file = std::make_shared(bytes); + const auto row_ranges = ::doris::RowRanges::create_single(2); + ScalarColumnReader reader(row_ranges, 2, chunk, nullptr, nullptr, nullptr); + ASSERT_TRUE(reader.init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) + .ok()); + auto dictionary_result = reader.dictionary_values(field.data_type); + ASSERT_TRUE(dictionary_result.has_value()) << dictionary_result.error(); + EXPECT_EQ(reader.dictionary_materialization_count_for_test(), 1); + + FilterMap filter; + ASSERT_TRUE(filter.init(nullptr, 2, false).ok()); + ColumnPtr ids = ColumnInt32::create(); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader.read_column_data(ids, field.data_type, nullptr, filter, 2, &rows, &eof, true) + .ok()); + ASSERT_EQ(rows, 2); + auto matched_values = reader.materialize_dictionary_values( + &assert_cast(*ids), field.data_type); + ASSERT_TRUE(matched_values.has_value()) << matched_values.error(); + EXPECT_EQ(reader.dictionary_materialization_count_for_test(), 1); + EXPECT_EQ(assert_cast(**matched_values).get_data(), + (ColumnInt32::Container {20, 10})); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryRepeatedRunsGatherDirectlyIntoDestination) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 1); + for (size_t row = 0; row < 64; ++row) { + encoder.Put(1); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 1; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 64, state).ok()); + EXPECT_EQ(state.dictionary_materialization_strategy, + ParquetDictionaryMaterializationStrategy::DIRECT); + ASSERT_EQ(output->size(), 64); + for (size_t row = 0; row < output->size(); ++row) { + EXPECT_EQ(assert_cast(*output).get_element(row), 20); + } +} + +TEST(ParquetV2NativeDecoderTest, DictionaryDirectGatherRollsBackLateCorruptRun) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (size_t row = 0; row < 8; ++row) { + encoder.Put(0); + } + for (size_t row = 0; row < 8; ++row) { + encoder.Put(3); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 2; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + const auto status = + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 16, state); + EXPECT_TRUE(status.is()) << status; + EXPECT_TRUE(output->empty()); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryIndexBoundsCheckHandlesVectorTailAndUnsignedIds) { + const std::vector valid {0, 7, 1, 6, 2, 5, 3, 4, 7, 0, 6}; + EXPECT_TRUE(native::dictionary_indices_in_bounds(valid.data(), valid.size(), 8)); + + auto invalid_tail = valid; + invalid_tail.back() = 8; + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_tail.data(), invalid_tail.size(), 8)); + + auto invalid_unsigned = valid; + invalid_unsigned[3] = std::numeric_limits::max(); + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_unsigned.data(), + invalid_unsigned.size(), 8)); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionRemapsConversionFailures) { + const std::vector dictionary {1, 1000, 2, 3}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeInt8 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(9); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_int32(dictionary, physical_ids, 7, null_runs, + filter, type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt8::Container {9, 0, 0, 2, 0})); + EXPECT_EQ(null_map, (NullMap {0, 1, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionExpandsStringsInPlace) { + const std::vector dictionary {"a", "bbb", "cc", "dddd"}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeString type; + auto column = type.create_column(); + assert_cast(*column).insert_data("prefix", 6); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_strings(dictionary, physical_ids, 7, null_runs, + filter, &column, &null_map, &statistics) + .ok()); + + ASSERT_EQ(column->size(), 5); + EXPECT_EQ(column->get_data_at(0).to_string_view(), "prefix"); + EXPECT_EQ(column->get_data_at(1).to_string_view(), ""); + EXPECT_EQ(column->get_data_at(2).to_string_view(), "bbb"); + EXPECT_EQ(column->get_data_at(3).to_string_view(), "cc"); + EXPECT_EQ(column->get_data_at(4).to_string_view(), ""); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionHandlesEmptyOutputShapes) { + DataTypeInt32 type; + ColumnChunkReaderStatistics statistics; + + auto all_null_column = type.create_column(); + assert_cast(*all_null_column).get_data().push_back(8); + NullMap all_null_map; + all_null_map.push_back(0); + const std::vector select_alternating {1, 0, 1, 0, 1}; + ASSERT_TRUE(materialize_selected_plain_int32({}, 5, {0, 5}, select_alternating, type, + &all_null_column, &all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto dictionary_all_null_column = type.create_column(); + assert_cast(*dictionary_all_null_column).get_data().push_back(8); + NullMap dictionary_all_null_map; + dictionary_all_null_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10}, {}, 5, {0, 5}, select_alternating, type, + &dictionary_all_null_column, + &dictionary_all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(dictionary_all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto all_filtered_column = type.create_column(); + assert_cast(*all_filtered_column).get_data().push_back(9); + NullMap all_filtered_map; + all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_plain_int32( + {10, 20, 30}, 5, {1, 1, 1, 1, 1}, std::vector(5, 0), type, + &all_filtered_column, &all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto dictionary_all_filtered_column = type.create_column(); + assert_cast(*dictionary_all_filtered_column).get_data().push_back(9); + NullMap dictionary_all_filtered_map; + dictionary_all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10, 20, 30}, {0, 1, 2}, 5, {1, 1, 1, 1, 1}, + std::vector(5, 0), type, + &dictionary_all_filtered_column, + &dictionary_all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(dictionary_all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); + const uint8_t* dictionary_address = dictionary.get(); + ByteArrayDictDecoder decoder; + + ASSERT_TRUE(decoder.set_dict(dictionary, dictionary_length, 2).ok()); + EXPECT_EQ(dictionary.get(), nullptr); + + RejectFixedConsumer fixed_consumer; + CaptureBinaryConsumer binary_consumer; + ASSERT_TRUE(decoder.decode_dictionary(fixed_consumer, binary_consumer).ok()); + ASSERT_EQ(binary_consumer.refs.size(), 2); + EXPECT_EQ(binary_consumer.refs[0].to_string_view(), "alpha"); + EXPECT_EQ(binary_consumer.refs[1].to_string_view(), "beta"); + EXPECT_EQ(binary_consumer.refs[0].data, + reinterpret_cast(dictionary_address + sizeof(uint32_t))); + + // bit width 1, an RLE run of three values (header = 3 << 1), dictionary id 1. + char valid_indices[] = {1, 6, 1}; + Slice valid_slice(valid_indices, sizeof(valid_indices)); + ASSERT_TRUE(decoder.set_data(&valid_slice).ok()); + std::vector decoded_indices; + ASSERT_TRUE(decoder.decode_dictionary_indices(3, &decoded_indices).ok()); + EXPECT_EQ(decoded_indices, std::vector({1, 1, 1})); + + // bit width 2, one RLE value with dictionary id 3. Skipping still validates the encoded id so + // filter selection cannot hide a corrupt dictionary stream. + char invalid_indices[] = {2, 2, 3}; + Slice invalid_slice(invalid_indices, sizeof(invalid_indices)); + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetDecodeSource& source = decoder; + EXPECT_TRUE(source.skip_values(1).is()); + + // Sparse decode must validate filtered dictionary ids too. Predicate selection must never + // turn a corrupt page into a successful read merely because the bad row was not selected. + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetSelection filtered_corrupt {.total_values = 1, .selected_values = 0, .ranges = {}}; + EXPECT_TRUE(decoder.decode_selected_dictionary_indices(filtered_corrupt, &decoded_indices) + .is()); + + Slice empty_indices; + EXPECT_TRUE(decoder.set_data(&empty_indices).is()); + + // Dictionary indices are uint32_t. Wider external bit widths would make the RLE decoder copy + // a five-byte repeated value into four-byte state before it can validate an index. + char oversized_bit_width[] = {33, 2, 0}; + Slice oversized_bit_width_slice(oversized_bit_width, sizeof(oversized_bit_width)); + EXPECT_TRUE(decoder.set_data(&oversized_bit_width_slice).is()); +} + +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryBoundsEntryCountBeforeReserve) { + auto dictionary = make_unique_buffer(1); + dictionary.get()[0] = 0; + ByteArrayDictDecoder decoder; + EXPECT_TRUE(decoder.set_dict(dictionary, 1, std::numeric_limits::max()) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, LegacyConvertedTimestampsRemainUtcAdjusted) { + TimezoneUtils::load_timezones_to_cache(); + for (const auto converted_type : + {tparquet::ConvertedType::TIMESTAMP_MILLIS, tparquet::ConvertedType::TIMESTAMP_MICROS}) { + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT64; + field.parquet_schema.__set_converted_type(converted_type); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.logical_type, ParquetLogicalType::TIMESTAMP); + EXPECT_TRUE(context.timestamp_is_adjusted_to_utc); + + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("Asia/Shanghai", shanghai)); + ASSERT_TRUE(init_decode_context_for_test(field, &shanghai, &context).ok()); + int64_t epoch = 0; + Slice epoch_slice(reinterpret_cast(&epoch), sizeof(epoch)); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(sizeof(epoch)); + ASSERT_TRUE(decoder->set_data(&epoch_slice).ok()); + ParquetMaterializationState state; + DataTypeDateTimeV2 type(0); + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, *decoder, context, 1, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 08:00:00"); + } +} + +TEST(ParquetV2NativeDecoderTest, GeospatialByteArrayAnnotationsDecodeAsRawBytes) { + const std::string wkb("\x01\x01\x00\x00\x00", 5); + for (const bool geometry : {true, false}) { + tparquet::LogicalType logical; + if (geometry) { + logical.__set_GEOMETRY(tparquet::GeometryType()); + } else { + logical.__set_GEOGRAPHY(tparquet::GeographyType()); + } + + NativeFieldSchema field; + field.name = geometry ? "geometry" : "geography"; + field.physical_type = tparquet::Type::BYTE_ARRAY; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.physical_type, ParquetPhysicalType::BYTE_ARRAY); + EXPECT_EQ(context.logical_type, ParquetLogicalType::NONE); + + DataTypeString type; + auto plain_column = type.create_column(); + auto encoded = encode_plain_byte_arrays({wkb}); + Slice plain_slice(encoded.data(), encoded.size()); + std::unique_ptr plain_decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, + plain_decoder) + .ok()); + ASSERT_TRUE(plain_decoder->set_data(&plain_slice).ok()); + ParquetMaterializationState plain_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*plain_column, *plain_decoder, context, 1, + plain_state) + .ok()); + ASSERT_EQ(plain_column->size(), 1); + EXPECT_EQ(plain_column->get_data_at(0).to_string_view(), wkb); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({wkb, "unused"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 2).ok()); + char dictionary_index[] = {1, 2, 0}; + Slice dictionary_slice(dictionary_index, sizeof(dictionary_index)); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + context.encoding = ParquetValueEncoding::DICTIONARY; + auto dictionary_column = type.create_column(); + ParquetMaterializationState dictionary_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, dictionary_decoder, + context, 1, dictionary_state) + .ok()); + ASSERT_EQ(dictionary_column->size(), 1); + EXPECT_EQ(dictionary_column->get_data_at(0).to_string_view(), wkb); + + field.physical_type = tparquet::Type::INT32; + EXPECT_TRUE( + init_decode_context_for_test(field, nullptr, &context).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, InvalidLogicalPhysicalPairsFailBeforeDecode) { + auto invalid = [](tparquet::Type::type physical, const tparquet::LogicalType& logical) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + + tparquet::LogicalType date; + date.__set_DATE(tparquet::DateType()); + EXPECT_FALSE(invalid(tparquet::Type::BOOLEAN, date).ok()); + + tparquet::LogicalType timestamp; + timestamp.__set_TIMESTAMP(tparquet::TimestampType()); + timestamp.TIMESTAMP.__set_unit(tparquet::TimeUnit()); + timestamp.TIMESTAMP.unit.__set_MICROS(tparquet::MicroSeconds()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, timestamp).ok()); + + tparquet::LogicalType integer; + integer.__set_INTEGER(tparquet::IntType()); + integer.INTEGER.__set_bitWidth(64); + integer.INTEGER.__set_isSigned(true); + EXPECT_FALSE(invalid(tparquet::Type::INT32, integer).ok()); + + tparquet::LogicalType string; + string.__set_STRING(tparquet::StringType()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, string).ok()); + + tparquet::LogicalType uuid; + uuid.__set_UUID(tparquet::UUIDType()); + EXPECT_FALSE(invalid(tparquet::Type::BYTE_ARRAY, uuid).ok()); + + auto invalid_converted = [](tparquet::Type::type physical, + tparquet::ConvertedType::type converted, int type_length = -1, + int precision = -1, int scale = -1) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_converted_type(converted); + if (type_length >= 0) { + field.parquet_schema.__set_type_length(type_length); + } + if (precision >= 0) { + field.parquet_schema.__set_precision(precision); + } + if (scale >= 0) { + field.parquet_schema.__set_scale(scale); + } + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UTF8).ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::TIME_MICROS).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UINT_64).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::ConvertedType::INTERVAL, 8) + .ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::DECIMAL, -1, 10, 2) + .ok()); + + NativeFieldSchema fixed; + fixed.name = "fixed"; + fixed.physical_type = tparquet::Type::FIXED_LEN_BYTE_ARRAY; + ParquetDecodeContext context; + EXPECT_FALSE(init_decode_context_for_test(fixed, nullptr, &context).ok()); + fixed.parquet_schema.__set_type_length(4); + EXPECT_TRUE(init_decode_context_for_test(fixed, nullptr, &context).ok()); +} + +TEST(ParquetV2NativeDecoderTest, NonStrictLegacyTimestampsKeepDefaultOnOverflow) { + DataTypePtr datetime_type = make_nullable(std::make_shared(6)); + NativeFieldSchema int96_field; + int96_field.physical_type = tparquet::Type::INT96; + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, false)); + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, true)); + + NativeFieldSchema utc_field; + utc_field.physical_type = tparquet::Type::INT64; + utc_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + utc_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + utc_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(utc_field, datetime_type, false)); + + NativeFieldSchema converted_utc_field; + converted_utc_field.physical_type = tparquet::Type::INT64; + converted_utc_field.parquet_schema.__set_converted_type( + tparquet::ConvertedType::TIMESTAMP_MICROS); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(converted_utc_field, datetime_type, + false)); + + NativeFieldSchema local_field = utc_field; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + EXPECT_FALSE( + preserves_timestamp_conversion_default_for_test(local_field, datetime_type, false)); + + NativeFieldSchema integer_field; + integer_field.physical_type = tparquet::Type::INT64; + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test( + integer_field, make_nullable(std::make_shared()), false)); +} + +TEST(ParquetV2NativeDecoderTest, FastInt96NormalizesLegacyOutOfDayNanos) { + EXPECT_TRUE(materialize_plain_int96({{-1, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{86400000000000LL, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{0, 2440588}, {-1, 2440588}}, {0, 1}).ok()); +} + +TEST(ParquetV2NativeDecoderTest, NonStrictLocalTimestampDefaultsBecomeNull) { + DataTypePtr datetime_type = make_nullable(std::make_shared(6)); + NativeFieldSchema local_field; + local_field.physical_type = tparquet::Type::INT64; + local_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + local_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + const uint64_t invalid_datetime = 0; + column->insert_data(reinterpret_cast(&invalid_datetime), sizeof(invalid_datetime)); + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 1); + + null_map[0] = 0; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 0); +} + +TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCursor) { + const ParquetSelection selection { + .total_values = 7, + .selected_values = 3, + .ranges = {{.first = 1, .count = 2}, {.first = 6, .count = 1}}}; + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {10, 11, 12, 13, 14, 15, 16, 17}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({11, 12, 16})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({17})); + + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded_strings = encode_plain_byte_arrays(strings); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + Slice string_slice(encoded_strings.data(), encoded_strings.size()); + ASSERT_TRUE(decoder->set_data(&string_slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 3); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "one"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "two"); + EXPECT_EQ(selected_strings.refs[2].to_string_view(), "six"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "seven"); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char booleans[] = {static_cast(0b10001101)}; + Slice boolean_slice(booleans, sizeof(booleans)); + ASSERT_TRUE(decoder->set_data(&boolean_slice).ok()); + CaptureFixedConsumer selected_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_booleans).ok()); + EXPECT_EQ(selected_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_boolean).ok()); + EXPECT_EQ(trailing_boolean.values(), std::vector({1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_booleans[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_booleans, sizeof(rle_booleans)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + CaptureFixedConsumer selected_rle_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_rle_booleans).ok()); + EXPECT_EQ(selected_rle_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_rle_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_rle_boolean).ok()); + EXPECT_EQ(trailing_rle_boolean.values(), std::vector({1})); +} + +TEST(ParquetV2NativeDecoderTest, PlainByteArrayPublishesOffsetsAndCoalescedSelectionSpans) { + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded = encode_plain_byte_arrays(strings); + Slice slice(encoded.data(), encoded.size()); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + + const ParquetSelection selection { + .total_values = 7, + .selected_values = 4, + .ranges = {{.first = 1, .count = 2}, {.first = 5, .count = 2}}}; + CapturePlainBinaryLayoutConsumer consumer; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, consumer).ok()); + EXPECT_FALSE(consumer.legacy_consume_called); + EXPECT_EQ(consumer.output_offsets, std::vector({0, 3, 6, 10, 13})); + ASSERT_EQ(consumer.source_offsets.size(), selection.selected_values); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[0], 3), "one"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[1], 3), "two"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[2], 4), "five"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[3], 3), "six"); + ASSERT_EQ(consumer.spans.size(), 2); + EXPECT_EQ(consumer.spans[0].first, 0); + EXPECT_EQ(consumer.spans[0].count, 2); + EXPECT_EQ(consumer.spans[1].first, 2); + EXPECT_EQ(consumer.spans[1].count, 2); + + CaptureBinaryConsumer trailing; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing).ok()); + ASSERT_EQ(trailing.refs.size(), 1); + EXPECT_EQ(trailing.refs[0].to_string_view(), "seven"); +} + +TEST(ParquetV2NativeDecoderTest, SparsePlainFixedDecodeDoesNotRetainGatherBuffer) { + constexpr size_t value_count = 1UL << 18; + std::vector integers(value_count); + std::iota(integers.begin(), integers.end(), 0); + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + const ParquetSelection selection { + .total_values = value_count, + .selected_values = 2, + .ranges = {{.first = 1, .count = 1}, {.first = value_count - 1, .count = 1}}}; + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), + std::vector({1, static_cast(value_count - 1)})); + // Sparse PLAIN spans can be consumed directly from the encoded page. Retaining a gather buffer + // makes a highly selective batch allocate in proportion to its selected width for no benefit. + EXPECT_EQ(decoder->retained_scratch_bytes(), 0); +} + +TEST(ParquetV2NativeDecoderTest, FixedPlainLargeSkipCannotWrapPageOffset) { + std::vector values {11, 22}; + Slice slice(reinterpret_cast(values.data()), values.size() * sizeof(int64_t)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int64_t)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(536870913).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({11})); +} + +TEST(ParquetV2NativeDecoderTest, PlainAndBooleanRleExposeRawValuesAndPreserveCursor) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {11, 22, 33}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer integer_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, integer_consumer).ok()); + EXPECT_EQ(integer_consumer.values(), std::vector({22, 33})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0b10001101)}; + Slice plain_boolean_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_boolean_slice).ok()); + CaptureFixedConsumer plain_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(8, plain_boolean_consumer).ok()); + EXPECT_EQ(plain_boolean_consumer.values(), + std::vector({1, 0, 1, 1, 0, 0, 0, 1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_boolean[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_boolean, sizeof(rle_boolean)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + ASSERT_TRUE(decoder->skip_values(3).ok()); + CaptureFixedConsumer rle_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(5, rle_boolean_consumer).ok()); + EXPECT_EQ(rle_boolean_consumer.values(), std::vector({1, 0, 0, 0, 1})); +} + +TEST(ParquetV2NativeDecoderTest, DeltaEncodingsExposeValuesAfterSkip) { + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer first_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, first_integer).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer remaining_integers; + ASSERT_TRUE(decoder->decode_fixed_values(2, remaining_integers).ok()); + EXPECT_EQ(first_integer.values(), std::vector({100})); + EXPECT_EQ(remaining_integers.values(), std::vector({99, 1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(3, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 3); + EXPECT_EQ(consumer.refs[0].to_string_view(), "prefix-b"); + EXPECT_EQ(consumer.refs[1].to_string_view(), "other"); + EXPECT_EQ(consumer.refs[2].to_string_view(), "other-tail"); + } +} + +TEST(ParquetV2NativeDecoderTest, DeltaBinaryPackedRejectsNonIntegralBlockGeometry) { + std::vector encoded(32); + uint8_t* cursor = encoded.data(); + cursor = encode_varint32(cursor, 3200); + cursor = encode_varint32(cursor, 33); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, 0); + encoded.resize(cursor - encoded.data()); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(1); + Slice slice(encoded.data(), encoded.size()); + EXPECT_FALSE(decoder->set_data(&slice).ok()); +} + +TEST(ParquetV2NativeDecoderTest, SparseStatefulEncodingsBatchDecodeAndCompact) { + const ParquetSelection selection { + .total_values = 3, + .selected_values = 2, + .ranges = {{.first = 0, .count = 1}, {.first = 2, .count = 1}}}; + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({100, 99})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 2); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "prefix-a"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "other"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "other-tail"); + } + + const std::vector floats {1.0F, -2.5F, 3.25F, 9.5F}; + std::vector encoded_floats(floats.size() * sizeof(float)); + for (size_t row = 0; row < floats.size(); ++row) { + const auto* bytes = reinterpret_cast(&floats[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded_floats[byte * floats.size() + row] = bytes[byte]; + } + } + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice float_slice(encoded_floats.data(), encoded_floats.size()); + ASSERT_TRUE(decoder->set_data(&float_slice).ok()); + CaptureFixedConsumer selected_floats; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_floats).ok()); + EXPECT_EQ(selected_floats.values(), std::vector({1.0F, 3.25F})); + CaptureFixedConsumer trailing_float; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_float).ok()); + EXPECT_EQ(trailing_float.values(), std::vector({9.5F})); +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRestoresFixedWidthRows) { + const std::vector values {1.0F, -2.5F, 3.25F}; + std::vector encoded(values.size() * sizeof(float)); + for (size_t row = 0; row < values.size(); ++row) { + const auto* bytes = reinterpret_cast(&values[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded[byte * values.size() + row] = bytes[byte]; + } + } + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({-2.5F, 3.25F})); +} + +TEST(ParquetV2NativeDecoderTest, BitPackedLevelCursorOperationsPreservePosition) { + char encoded[] = {static_cast(0b00001101)}; + Slice levels(encoded, sizeof(encoded)); + LevelDecoder decoder; + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 1, 4).ok()); + + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 4), 1); + EXPECT_EQ(value, 1); + EXPECT_EQ(decoder.get_next(), 0); + decoder.rewind_one(); + EXPECT_EQ(decoder.get_next(), 0); + level_t tail[2] = {-1, -1}; + EXPECT_EQ(decoder.get_levels(tail, 2), 2); + EXPECT_EQ(tail[0], 1); + EXPECT_EQ(tail[1], 1); + + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice truncated_levels(truncated_rle, sizeof(truncated_rle)); + LevelDecoder rle_decoder; + ASSERT_TRUE(rle_decoder.init(&truncated_levels, tparquet::Encoding::RLE, 1, 1).ok()); + level_t truncated_value = -1; + EXPECT_EQ(rle_decoder.get_levels(&truncated_value, 1), 0); +} + +TEST(ParquetV2NativeDecoderTest, BitPackedLevelByteCountDoesNotWrapAtLargeCounts) { + char placeholder[8] = {}; + constexpr uint32_t num_levels = 1'500'000'000; + constexpr size_t expected_bytes = 562'500'000; + Slice levels(placeholder, expected_bytes); + LevelDecoder decoder; + + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 4, num_levels).ok()); + EXPECT_EQ(levels.size, 0); +} + +TEST(ParquetV2NativeDecoderTest, RejectsLevelsAboveSchemaMaximumOnEveryDecodePath) { + auto make_decoder = [](tparquet::Encoding::type encoding) { + static char encoded[] = {0x03}; // width=2 value=3, while the schema maximum is 2. + static char rle_encoded[] = {0x02, 0x00, 0x00, 0x00, 0x02, 0x03}; + Slice levels = encoding == tparquet::Encoding::BIT_PACKED + ? Slice(encoded, sizeof(encoded)) + : Slice(rle_encoded, sizeof(rle_encoded)); + LevelDecoder decoder; + EXPECT_TRUE(decoder.init(&levels, encoding, 2, 1).ok()); + return decoder; + }; + + for (auto encoding : {tparquet::Encoding::BIT_PACKED, tparquet::Encoding::RLE}) { + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_levels(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + EXPECT_EQ(decoder.get_next(), -1); + } + } +} + +TEST(ParquetV2NativeDecoderTest, NestedReadersRejectRleAndBitPackedLevelsAboveMaximum) { + for (auto encoding : {tparquet::Encoding::RLE, tparquet::Encoding::BIT_PACKED}) { + const std::vector payload = encoding == tparquet::Encoding::RLE + ? std::vector {2, 0, 0, 0, 2, 3} + : std::vector {3}; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + + EXPECT_TRUE(load_malformed_nested_page(header, payload, 1, 2).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, TruncatedBooleanStreamsFailWhileSkipping) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0xFF)}; + Slice plain_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_slice).ok()); + EXPECT_FALSE(decoder->skip_values(9).ok()); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + // A bit-packed run header for eight values without its required payload byte. + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice rle_slice(truncated_rle, sizeof(truncated_rle)); + ASSERT_TRUE(decoder->set_data(&rle_slice).ok()); + EXPECT_FALSE(decoder->skip_values(1).ok()); + + // The first literal group is complete but the second has only its header. Exact-count checks + // must reject both dense and selected decodes instead of exposing the retained vector tail. + char partially_truncated_rle[] = {0x03, 0x00, 0x00, 0x00, 0x03, static_cast(0xFF), 0x03}; + Slice partially_truncated_slice(partially_truncated_rle, sizeof(partially_truncated_rle)); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer dense_consumer; + EXPECT_FALSE(decoder->decode_fixed_values(9, dense_consumer).ok()); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_tail { + .total_values = 9, .selected_values = 1, .ranges = {{.first = 8, .count = 1}}}; + EXPECT_FALSE(decoder->decode_selected_fixed_values(select_tail, selected_consumer).ok()); +} + +TEST(ParquetV2NativeDecoderTest, CompactRleSkipsKeepScratchBounded) { + constexpr uint32_t value_count = 1U << 20; + uint8_t header[16]; + uint8_t* header_end = encode_varint32(header, value_count << 1); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"only"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 1).ok()); + std::vector dictionary_indices {0}; + dictionary_indices.insert(dictionary_indices.end(), reinterpret_cast(header), + reinterpret_cast(header_end)); + Slice dictionary_slice(dictionary_indices.data(), dictionary_indices.size()); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + ParquetDecodeSource& dictionary_source = dictionary_decoder; + ASSERT_TRUE(dictionary_source.skip_values(value_count).ok()); + EXPECT_LE(dictionary_decoder.retained_scratch_bytes(), 4096 * sizeof(uint32_t)); + + std::vector boolean_payload(reinterpret_cast(header), + reinterpret_cast(header_end)); + boolean_payload.push_back(0); + std::vector boolean_page(sizeof(uint32_t) + boolean_payload.size()); + encode_fixed32_le(reinterpret_cast(boolean_page.data()), boolean_payload.size()); + memcpy(boolean_page.data() + sizeof(uint32_t), boolean_payload.data(), boolean_payload.size()); + Slice boolean_slice(boolean_page.data(), boolean_page.size()); + std::unique_ptr boolean_decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, boolean_decoder) + .ok()); + ASSERT_TRUE(boolean_decoder->set_data(&boolean_slice).ok()); + ASSERT_TRUE(boolean_decoder->skip_values(value_count).ok()); + EXPECT_LE(boolean_decoder->retained_scratch_bytes(), 4096); +} + +TEST(ParquetV2NativeDecoderTest, CompactDeltaSkipKeepsScratchBounded) { + constexpr uint32_t delta_count = 1U << 20; + std::vector encoded(64); + uint8_t* cursor = encoded.data(); + cursor = encode_varint32(cursor, delta_count); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, delta_count + 1); + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta + *cursor++ = 0; // zero-width miniblock + encoded.resize(cursor - encoded.data()); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(delta_count + 1); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(delta_count + 1).ok()); + EXPECT_LE(decoder->retained_scratch_bytes(), 4096 * sizeof(int32_t) + 1); +} + +TEST(ParquetV2NativeDecoderTest, DeltaPaddingBitCountIsWidenedBeforeCursorAdvance) { + int64_t padding_bits = 0; + ASSERT_TRUE(detail::checked_delta_padding_bits(64, std::numeric_limits::max(), + &padding_bits) + .ok()); + EXPECT_EQ(padding_bits, 64LL * std::numeric_limits::max()); + EXPECT_GT(padding_bits, std::numeric_limits::max()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArraySkipsKeepScratchBounded) { + constexpr size_t value_count = 1U << 16; + std::vector<::parquet::ByteArray> values(value_count); + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + static_cast(encoding), decoder) + .ok()); + decoder->set_expected_values(value_count); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(value_count).ok()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); + } +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArrayLongPrefixSparseWorkIsByteBounded) { + constexpr size_t value_count = 128; + const std::string repeated_value(256UL << 10, 'x'); + std::vector<::parquet::ByteArray> values( + value_count, + ::parquet::ByteArray(static_cast(repeated_value.size()), + reinterpret_cast(repeated_value.data()))); + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + + auto make_decoder = [&]() { + std::unique_ptr decoder; + EXPECT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(value_count); + Slice slice(encoded->data(), encoded->size()); + EXPECT_TRUE(decoder->set_data(&slice).ok()); + return decoder; + }; + + auto skipped = make_decoder(); + ASSERT_TRUE(skipped->skip_values(value_count).ok()); + EXPECT_LT(skipped->retained_scratch_bytes(), 8UL << 20); + + auto sparse = make_decoder(); + CaptureBinaryConsumer consumer; + const ParquetSelection selection { + .total_values = value_count, + .selected_values = 1, + .ranges = {{.first = value_count - 1, .count = 1}}, + }; + ASSERT_TRUE(sparse->decode_selected_binary_values(selection, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 1); + EXPECT_EQ(consumer.refs[0].to_string_view(), repeated_value); + EXPECT_LT(sparse->retained_scratch_bytes(), 8UL << 20); +} + +TEST(ParquetV2NativeDecoderTest, DeltaFixedWidthValidatesFilteredAndSkippedValues) { + const std::vector values {"good", "bad"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_type_length(4); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_good { + .total_values = 2, .selected_values = 1, .ranges = {{.first = 0, .count = 1}}}; + EXPECT_TRUE(decoder->decode_selected_fixed_values(select_good, selected_consumer) + .is()); + + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_TRUE(decoder->skip_values(2).is()); +} + +TEST(ParquetV2NativeDecoderTest, TruncatedFixedWidthPlainKeepsPublicErrorKeyword) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(4); + char truncated[] = {'a', 'b', 'c'}; + Slice slice(truncated, sizeof(truncated)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + const auto status = decoder->decode_fixed_values(1, consumer); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Unexpected end of stream"), std::string::npos); +} + +TEST(ParquetV2NativeDecoderTest, DeltaSkipRequiresTheRequestedValueCount) { + const std::vector integers {7}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(integers.data(), static_cast(integers.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(2).ok()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaHeadersAndLengthsAreBoundedBeforeAllocation) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(1); + // block_size=128, miniblocks=4, total_values=INT_MAX, first_value=0. + char oversized_count[] = {static_cast(0x80), + 0x01, + 0x04, + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + 0x07, + 0x00}; + Slice oversized_count_slice(oversized_count, sizeof(oversized_count)); + EXPECT_TRUE(decoder->set_data(&oversized_count_slice).is()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + // A single decoded length of 1 GiB with no following payload bytes must fail before the + // decoder resizes its backing data buffer. + char oversized_length[] = {static_cast(0x80), + 0x01, + 0x04, + 0x01, + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + 0x08}; + Slice oversized_length_slice(oversized_length, sizeof(oversized_length)); + CaptureBinaryConsumer consumer; + EXPECT_TRUE(decoder->set_data(&oversized_length_slice).is()); + EXPECT_TRUE(consumer.refs.empty()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + std::vector huge_prefix(64); + uint8_t* cursor = huge_prefix.data(); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, std::numeric_limits::max() - 1); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, 0); // one empty suffix + huge_prefix.resize(cursor - huge_prefix.data()); + Slice huge_prefix_slice(huge_prefix.data(), huge_prefix.size()); + ASSERT_TRUE(decoder->set_data(&huge_prefix_slice).ok()); + EXPECT_TRUE(decoder->decode_binary_values(1, consumer).is()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); +} + +TEST(ParquetV2NativeDecoderTest, EmptyDeltaLengthPageResetsDecoderState) { + const std::vector strings {"old", "state"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + Slice valid(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&valid).ok()); + Slice empty; + EXPECT_TRUE(decoder->set_data(&empty).is()); +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRejectsPartialRowsAtPageBoundary) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + char partial_row[] = {0, 1, 2, 3, 4}; + Slice slice(partial_row, sizeof(partial_row)); + EXPECT_TRUE(decoder->set_data(&slice).is()); +} + +Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t value_rows, + size_t key_values, size_t value_values, + std::vector key_rep_levels, std::vector value_rep_levels, + bool value_eof, std::vector key_def_levels = {}, + std::vector value_def_levels = {}) { + auto value_type = make_nullable(std::make_shared()); + auto map_type = std::make_shared(key_type, value_type); + ColumnPtr column = map_type->create_column(); + NativeFieldSchema field; + field.name = "m"; + field.data_type = map_type; + field.definition_level = 1; + field.repetition_level = 1; + field.repeated_parent_def_level = 0; + + if (key_def_levels.empty()) { + key_def_levels.assign(key_rep_levels.size(), 1); + } + if (value_def_levels.empty()) { + value_def_levels.assign(value_rep_levels.size(), 1); + } + auto key_reader = std::make_unique( + key_rows, key_values, true, std::move(key_rep_levels), std::move(key_def_levels)); + auto value_reader = std::make_unique(value_rows, value_values, value_eof, + std::move(value_rep_levels), + std::move(value_def_levels)); + MapColumnReader reader(scripted_row_ranges(), key_rows, nullptr, nullptr); + RETURN_IF_ERROR(reader.init(std::move(key_reader), std::move(value_reader), &field)); + + auto root = std::make_shared(std::make_shared(), + std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + return reader.read_column_data(column, map_type, root, filter, key_rows, &read_rows, &eof, + false); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderUsesKeyShapeForNestedValues) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {0, 1}, {0, 2, 1}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryDistribution) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 4, 4, {0, 1, 0, 1}, {0, 1, 1, 0}, true) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryPresence) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 2, 2, {0, 0}, {0, 0}, true, {0, 1}, {1, 0}) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, ComplexReadersRejectMalformedSiblingCounts) { + auto int_type = std::make_shared(); + EXPECT_TRUE( + read_scripted_map(int_type, 2, 1, 2, 1, {0, 0}, {0}, true).is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderPreservesNullableKeysWrittenByDoris) { + auto nullable_key = make_nullable(std::make_shared()); + EXPECT_TRUE(read_scripted_map(nullable_key, 1, 1, 1, 1, {0}, {0}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShortSibling) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 0}); + children["b"] = std::make_unique(1, 1, true, std::vector {0}, + std::vector {0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedRepeatedParentShape) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.repetition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique( + 2, 3, true, std::vector {0, 1, 0}, std::vector {0, 0, 0}); + children["b"] = std::make_unique( + 2, 3, true, std::vector {0, 0, 1}, std::vector {0, 0, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedOptionalParentPresence) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.definition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 1}); + children["b"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {1, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { + constexpr size_t value_count = 1UL << 20; + std::vector encoded(value_count * sizeof(float), 0); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(value_count, consumer).ok()); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + EXPECT_EQ(decoder->active_scratch_bytes(), value_count * sizeof(float)); + + std::vector ordinary_encoded(sizeof(float), 0); + Slice ordinary_slice(ordinary_encoded.data(), ordinary_encoded.size()); + ASSERT_TRUE(decoder->set_data(&ordinary_slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + // Capacity records the high-water allocation while size records this batch. The reader needs + // both to distinguish stable large batches from a one-off outlier before releasing capacity. + EXPECT_EQ(decoder->active_scratch_bytes(), sizeof(float)); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + decoder->release_scratch(64UL << 10); + EXPECT_LE(decoder->retained_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, DecompressionScratchStaysActiveUntilPageExhaustion) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + auto compressed_page = [&](uint32_t value_count) { + std::vector encoded_levels(8); + uint8_t* level_end = encode_varint32(encoded_levels.data(), value_count << 1); + *level_end++ = 1; + encoded_levels.resize(level_end - encoded_levels.data()); + std::vector payload(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), encoded_levels.size()); + payload.insert(payload.end(), encoded_levels.begin(), encoded_levels.end()); + payload.resize(payload.size() + static_cast(value_count) * sizeof(int32_t)); + + faststring compressed; + DORIS_CHECK(codec->compress(Slice(payload.data(), payload.size()), &compressed).ok()); + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(value_count); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, std::vector(compressed.data(), + compressed.data() + compressed.size())); + }; + + constexpr uint32_t LARGE_VALUE_COUNT = 1U << 20; + auto bytes = compressed_page(LARGE_VALUE_COUNT); + const auto ordinary_page = compressed_page(1); + bytes.insert(bytes.end(), ordinary_page.begin(), ordinary_page.end()); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(static_cast(LARGE_VALUE_COUNT) + 1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.definition_level = 1; + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, + static_cast(LARGE_VALUE_COUNT) + 1, nullptr, + context); + ASSERT_TRUE(reader.init().ok()); + ASSERT_TRUE(reader.load_page_data().ok()); + ASSERT_GT(reader.active_decoder_scratch_bytes(), 1UL << 20); + ASSERT_TRUE(reader.skip_values(LARGE_VALUE_COUNT).ok()); + ASSERT_TRUE(reader.next_page().ok()); + ASSERT_TRUE(reader.parse_page_header().ok()); + ASSERT_TRUE(reader.load_page_data().ok()); + EXPECT_LT(reader.active_decoder_scratch_bytes(), 64UL << 10); + const size_t retained_while_active = reader.retained_decoder_scratch_bytes(); + ASSERT_GT(retained_while_active, 1UL << 20); + reader.release_decoder_scratch(64UL << 10); + EXPECT_EQ(reader.retained_decoder_scratch_bytes(), retained_while_active); + + ASSERT_TRUE(reader.skip_values(1).ok()); + ASSERT_TRUE(reader.next_page().ok()); + // A release requested while a decoder points into the buffer must execute at the next safe + // page boundary; otherwise periodic probes can never age out a previous large-page capacity. + EXPECT_LE(reader.retained_decoder_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArrayScratchReleasePreservesPrefixState) { + const std::vector values {std::string(4096, 'x'), "shared-prefix-a", + "shared-prefix-b", "shared-prefix-c"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + decoder->release_scratch(64); + ASSERT_TRUE(decoder->decode_binary_values(2, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 2); + EXPECT_EQ(consumer.refs[0].to_string_view(), values[2]); + EXPECT_EQ(consumer.refs[1].to_string_view(), values[3]); +} + +TEST(ParquetV2NativeDecoderTest, DeltaBitPackScratchReleasePreservesMiniblockState) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + + std::vector outlier_header(64); + uint8_t* cursor = outlier_header.data(); + cursor = encode_varint32(cursor, 512); // values per block + cursor = encode_varint32(cursor, 16); // miniblocks per block + cursor = encode_varint32(cursor, 2); // total values + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta, zig-zag encoded + memset(cursor, 0, 16); // one zero bit width per miniblock + cursor += 16; + outlier_header.resize(cursor - outlier_header.data()); + decoder->set_expected_values(2); + Slice outlier_slice(outlier_header.data(), outlier_header.size()); + ASSERT_TRUE(decoder->set_data(&outlier_slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + + std::vector values(40); + std::iota(values.begin(), values.end(), 0); + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + decoder->release_scratch(8); + ASSERT_TRUE(decoder->decode_fixed_values(values.size() - 1, consumer).ok()); + EXPECT_EQ(consumer.values().size(), values.size() + 1); +} + +TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption) { + auto parse_header = [](tparquet::PageHeader header) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + if (header.compressed_page_size > 0) { + bytes.resize(bytes.size() + header.compressed_page_size); + } + MemoryBufferedReader reader(bytes); + tparquet::ColumnMetaData metadata; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, bytes.size(), 1, metadata, + context); + return page_reader.parse_page_header(); + }; + + tparquet::PageHeader negative; + negative.type = tparquet::PageType::DATA_PAGE; + negative.__set_compressed_page_size(-1); + negative.__set_uncompressed_page_size(1); + negative.__isset.data_page_header = true; + negative.data_page_header.__set_num_values(1); + EXPECT_TRUE(parse_header(negative).is()); + + tparquet::PageHeader oversized_levels; + oversized_levels.type = tparquet::PageType::DATA_PAGE_V2; + oversized_levels.__set_compressed_page_size(1); + oversized_levels.__set_uncompressed_page_size(1); + oversized_levels.__isset.data_page_header_v2 = true; + oversized_levels.data_page_header_v2.__set_num_values(1); + oversized_levels.data_page_header_v2.__set_num_rows(1); + oversized_levels.data_page_header_v2.__set_repetition_levels_byte_length(2); + oversized_levels.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(oversized_levels).is()); + + tparquet::PageHeader impossible_counts = oversized_levels; + impossible_counts.__set_compressed_page_size(0); + impossible_counts.__set_uncompressed_page_size(0); + impossible_counts.data_page_header_v2.__set_repetition_levels_byte_length(0); + impossible_counts.data_page_header_v2.__set_num_nulls(2); + EXPECT_TRUE(parse_header(impossible_counts).is()); + + tparquet::PageHeader missing_layout; + missing_layout.type = tparquet::PageType::DATA_PAGE; + missing_layout.__set_compressed_page_size(0); + missing_layout.__set_uncompressed_page_size(0); + EXPECT_TRUE(parse_header(missing_layout).is()); + + auto swapped_layout = oversized_levels; + swapped_layout.type = tparquet::PageType::DATA_PAGE; + EXPECT_TRUE(parse_header(swapped_layout).is()); + + auto competing_layouts = negative; + competing_layouts.__set_compressed_page_size(0); + competing_layouts.__set_uncompressed_page_size(0); + competing_layouts.__isset.data_page_header_v2 = true; + competing_layouts.data_page_header_v2.__set_num_values(0); + competing_layouts.data_page_header_v2.__set_num_rows(0); + competing_layouts.data_page_header_v2.__set_num_nulls(0); + competing_layouts.data_page_header_v2.__set_repetition_levels_byte_length(0); + competing_layouts.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(competing_layouts).is()); +} + +TEST(ParquetV2NativeDecoderTest, ShiftedOffsetIndexFallsBackToSequentialPages) { + auto data_page = [](int32_t value) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(sizeof(value)); + header.__set_uncompressed_page_size(sizeof(value)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(sizeof(value)); + memcpy(payload.data(), &value, sizeof(value)); + return serialize_page(header, payload); + }; + auto verify_fallback = [&](bool cache_hit) { + const auto first_page = data_page(10); + const auto second_page = data_page(20); + std::vector bytes = first_page; + bytes.insert(bytes.end(), second_page.begin(), second_page.end()); + bytes.push_back(0); + + tparquet::OffsetIndex offset_index; + tparquet::PageLocation first_location; + first_location.__set_offset(0); + first_location.__set_compressed_page_size(cast_set(first_page.size() + 1)); + first_location.__set_first_row_index(0); + tparquet::PageLocation second_location; + second_location.__set_offset(cast_set(first_page.size() + 1)); + second_location.__set_compressed_page_size(cast_set(second_page.size())); + second_location.__set_first_row_index(1); + offset_index.__set_page_locations({first_location, second_location}); + EXPECT_TRUE( + validate_offset_index(offset_index, {.offset = 0, .length = bytes.size()}, 0, 2)); + + const std::string cache_key = cache_hit ? "shifted-index-cache-hit" : ""; + if (cache_hit) { + auto* page = new DataPage(first_page.size(), true, segment_v2::DATA_PAGE); + memcpy(page->data(), first_page.data(), first_page.size()); + page->reset_size(first_page.size()); + PageCacheHandle handle; + StoragePageCache::instance()->insert( + StoragePageCache::CacheKey(cache_key, bytes.size(), 0), page, &handle, + segment_v2::DATA_PAGE); + } + + tparquet::ColumnMetaData metadata; + metadata.__set_type(tparquet::Type::INT32); + metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + metadata.__set_num_values(2); + metadata.__set_total_compressed_size(bytes.size()); + metadata.__set_data_page_offset(0); + MemoryBufferedReader stream(std::move(bytes)); + PageReader reader(&stream, nullptr, 0, metadata.total_compressed_size, 2, + metadata, ParquetPageReadContext(cache_hit, cache_key), + &offset_index); + + ASSERT_TRUE(reader.parse_page_header().ok()); + reader.skip_page_data(); + ASSERT_TRUE(reader.next_page().ok()); + // The rectangles above are non-overlapping but shifted one byte from the serialized pages. + // Discarding the optional index must continue from the first page's real payload end. + ASSERT_TRUE(reader.parse_page_header().ok()); + const tparquet::PageHeader* parsed = nullptr; + ASSERT_TRUE(reader.get_page_header(&parsed).ok()); + EXPECT_EQ(parsed->data_page_header.num_values, 1); + }; + verify_fallback(false); + verify_fallback(true); +} + +TEST(ParquetV2NativeDecoderTest, FlatPagesRejectLogicalAndPhysicalCardinalityMismatch) { + auto init_chunk = [](tparquet::PageHeader header, bool with_offset_index) { + std::vector payload(static_cast(header.compressed_page_size), 0); + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(bytes.size()); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + if (with_offset_index) { + ColumnChunkReader reader_with_index(&reader, &chunk, &field, &offset_index, + 1, nullptr, context); + return reader_with_index.init(); + } + ColumnChunkReader sequential_reader(&reader, &chunk, &field, nullptr, 1, + nullptr, context); + return sequential_reader.init(); + }; + + tparquet::PageHeader v2; + v2.type = tparquet::PageType::DATA_PAGE_V2; + v2.__set_compressed_page_size(8); + v2.__set_uncompressed_page_size(8); + v2.__isset.data_page_header_v2 = true; + v2.data_page_header_v2.__set_num_values(2); + v2.data_page_header_v2.__set_num_rows(1); + v2.data_page_header_v2.__set_num_nulls(0); + v2.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + v2.data_page_header_v2.__set_repetition_levels_byte_length(0); + v2.data_page_header_v2.__set_definition_levels_byte_length(0); + v2.data_page_header_v2.__set_is_compressed(false); + auto status = init_chunk(v2, false); + EXPECT_TRUE(status.is()) << status; + status = init_chunk(v2, true); + EXPECT_TRUE(status.is()) << status; + + tparquet::PageHeader v1; + v1.type = tparquet::PageType::DATA_PAGE; + v1.__set_compressed_page_size(8); + v1.__set_uncompressed_page_size(8); + v1.__isset.data_page_header = true; + v1.data_page_header.__set_num_values(2); + v1.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + v1.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + v1.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + status = init_chunk(v1, true); + EXPECT_TRUE(status.is()) << status; +} + +TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsOffsetIndexRowSpanMismatch) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(8); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(2); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + auto bytes = serialize_page(header, std::vector(8)); + MemoryBufferedReader stream(bytes); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + field.definition_level = 1; + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(bytes.size()); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + ParquetPageReadContext context(false, ""); + + ColumnChunkReader reader(&stream, &chunk, &field, &offset_index, + /*total_rows=*/2, nullptr, context); + const auto status = reader.init(); + EXPECT_TRUE(status.is()) << status; +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyAllNullPagesInitializeZeroValueDecoders) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, true); + EXPECT_TRUE(status.ok()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyPagesRejectDefinitionLevelsThatRequireValues) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, false); + EXPECT_TRUE(status.is()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyReaderSkipsLeadingZeroValuePages) { + auto page = [](bool v2, std::optional value) { + std::vector payload; + if (value.has_value()) { + const std::vector definition_levels {2, 1}; + if (v2) { + payload = definition_levels; + } else { + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), definition_levels.size()); + payload.insert(payload.end(), definition_levels.begin(), definition_levels.end()); + } + const auto* value_bytes = reinterpret_cast(&*value); + payload.insert(payload.end(), value_bytes, value_bytes + sizeof(*value)); + } else if (!v2) { + // V1 prefixes its RLE definition-level stream even when that stream has no values. + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), 0); + } + tparquet::PageHeader header; + header.type = v2 ? tparquet::PageType::DATA_PAGE_V2 : tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + if (v2) { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(value.has_value() ? 1 : 0); + header.data_page_header_v2.__set_num_rows(value.has_value() ? 1 : 0); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_definition_levels_byte_length(value.has_value() ? 2 + : 0); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } else { + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(value.has_value() ? 1 : 0); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + } + return serialize_page(header, payload); + }; + + static const auto utc = cctz::utc_time_zone(); + for (const bool v2 : {false, true}) { + SCOPED_TRACE(v2 ? "v2" : "v1"); + auto bytes = page(v2, std::nullopt); + const auto nonempty_page = page(v2, 7); + bytes.insert(bytes.end(), nonempty_page.begin(), nonempty_page.end()); + auto file = std::make_shared(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.data_type = make_nullable(std::make_shared()); + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + auto row_ranges = ::doris::RowRanges::create_single(1); + ScalarColumnReader scalar_reader(row_ranges, 1, chunk, nullptr, &utc, + nullptr); + ASSERT_TRUE( + scalar_reader + .init(file, &field, bytes.size(), nullptr, "", ParquetReaderCompat {}, true) + .ok()); + FilterMap filter; + ASSERT_TRUE(filter.init(nullptr, 1, false).ok()); + ColumnPtr scalar_column = field.data_type->create_column(); + size_t scalar_rows = 0; + bool scalar_eof = false; + while (scalar_rows == 0 && !scalar_eof) { + size_t read_now = 0; + ASSERT_TRUE(scalar_reader + .read_column_data(scalar_column, field.data_type, nullptr, filter, + 1, &read_now, &scalar_eof, false) + .ok()); + scalar_rows += read_now; + } + ASSERT_EQ(scalar_rows, 1); + const auto& scalar_nullable = assert_cast(*scalar_column); + EXPECT_FALSE(scalar_nullable.is_null_at(0)); + EXPECT_EQ( + assert_cast(scalar_nullable.get_nested_column()).get_element(0), + 7); + + std::unique_ptr reader; + ASSERT_TRUE(LevelReader::create(file, chunk, &field, 1, bytes.size(), nullptr, false, "", + ParquetReaderCompat {}, &reader) + .ok()); + + std::vector repetition_levels; + std::vector definition_levels; + size_t rows_read = 0; + const auto status = + reader->read_rows(1, &repetition_levels, &definition_levels, &rows_read); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(rows_read, 1); + EXPECT_EQ(repetition_levels, (std::vector {0})); + EXPECT_EQ(definition_levels, (std::vector {1})); + } +} + +TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsMissingAdvertisedRowStarts) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(12); + header.__set_uncompressed_page_size(12); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(2); + header.data_page_header_v2.__set_num_rows(2); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(4); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + // Two values [0, 1] contain only one repetition-level zero, so they describe one row. + EXPECT_TRUE(load_malformed_nested_page(header, {2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0}, 2) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, FirstNestedV1PageRejectsOrphanContinuation) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + // A two-value RLE run at repetition level 1 has no level-0 row start. + const std::vector payload {2, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0}; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(2); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + EXPECT_TRUE(load_malformed_nested_page(header, payload, 2).is()); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1ContinuationRemainsValidAfterFirstRowStart) { + auto make_page = [](int values, std::vector payload) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, payload); + }; + auto first_page = make_page(2, {2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0}); + auto second_page = make_page(1, {2, 0, 0, 0, 2, 1, 0, 0, 0, 0}); + first_page.insert(first_page.end(), second_page.begin(), second_page.end()); + + MemoryBufferedReader reader(first_page); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(3); + chunk.meta_data.__set_total_compressed_size(first_page.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + ASSERT_TRUE(cross_page); + ASSERT_TRUE(chunk_reader.load_cross_page_nested_row(levels, &cross_page).ok()); + EXPECT_FALSE(cross_page); + EXPECT_EQ(levels, std::vector({0, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1IgnoresUnverifiableOffsetIndexRows) { + auto make_page = [](int values, std::vector payload) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, payload); + }; + const auto first_page = make_page(2, {2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0}); + const auto second_page = make_page(1, {2, 0, 0, 0, 2, 1, 0, 0, 0, 0}); + std::vector bytes = first_page; + bytes.insert(bytes.end(), second_page.begin(), second_page.end()); + + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(3); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + tparquet::OffsetIndex offset_index; + tparquet::PageLocation first_location; + first_location.__set_offset(0); + first_location.__set_compressed_page_size(first_page.size()); + first_location.__set_first_row_index(0); + tparquet::PageLocation second_location; + second_location.__set_offset(first_page.size()); + second_location.__set_compressed_page_size(second_page.size()); + // This V1 page continues row zero, but its index claims a new logical row. Repetition levels + // are the only authoritative source, so indexed seeking must be disabled for this chunk. + second_location.__set_first_row_index(1); + offset_index.__set_page_locations({first_location, second_location}); + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&stream, &chunk, &field, &offset_index, 1, nullptr, + context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + ASSERT_TRUE(cross_page); + ASSERT_TRUE(chunk_reader.load_cross_page_nested_row(levels, &cross_page).ok()); + EXPECT_FALSE(cross_page); + EXPECT_EQ(levels, std::vector({0, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1DiscardedOffsetIndexStopsAtLogicalChunkEnd) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + const std::vector payload {2, 0, 0, 0, 2, 0, 0, 0, 0, 0}; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + auto bytes = serialize_page(header, payload); + const size_t page_size = bytes.size(); + // A compatibility reader may pad the validated physical range beyond the encoded chunk. + bytes.resize(page_size + 16, 0); + + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(page_size); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(page_size); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + ColumnChunkRange padded_range {.offset = 0, .length = bytes.size()}; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&stream, &chunk, &field, &offset_index, 1, nullptr, + context, &padded_range); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + EXPECT_EQ(rows, 1); + EXPECT_FALSE(cross_page); + EXPECT_FALSE(chunk_reader.has_next_page()); +} + +TEST(ParquetV2NativeDecoderTest, HugeNestedPageCountsDoNotPreallocateFromHeaders) { + for (auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(std::numeric_limits::max()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(std::numeric_limits::max()); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } + // The four-byte V1 level length (or V2 value payload) contains no advertised levels. The + // reader must report corruption without reserving INT_MAX level slots first. + EXPECT_TRUE(load_malformed_nested_page(header, {0, 0, 0, 0}).is()); + } +} + +TEST(ParquetV2NativeDecoderTest, PageDecompressionRejectsBothSizeMismatchDirections) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + std::vector input(8, 7); + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(input.data(), input.size()), &compressed).ok()); + std::vector payload(compressed.data(), compressed.data() + compressed.size()); + + for (const int32_t advertised_size : {4, 12}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(advertised_size); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + EXPECT_FALSE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY).ok()); + } +} + +TEST(ParquetV2NativeDecoderTest, UncompressedDictionaryRequiresEqualPhysicalAndLogicalSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(1); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE(load_scripted_page(header, std::vector(8), + tparquet::CompressionCodec::UNCOMPRESSED) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, EmptyDictionaryRejectsDeclaredPayloadBeforeAllocation) { + tparquet::PageHeader header; + header.__set_uncompressed_page_size(std::numeric_limits::max()); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(0); + EXPECT_TRUE(validate_dictionary_page_size(header).is()); +} + +TEST(ParquetV2NativeDecoderTest, NonemptyFixedWidthDictionaryRejectsExtentBeforeAllocation) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::array value {}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(8 << 20); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE(validate_dictionary_page_size(header, sizeof(int32_t)).is()); + + const std::vector payload(compressed.data(), compressed.data() + compressed.size()); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, RequiredFixedWidthPageRejectsImpossibleExtentBeforeAllocation) { + for (const auto encoding : {tparquet::Encoding::PLAIN, tparquet::Encoding::BYTE_STREAM_SPLIT}) { + tparquet::PageHeader header; + header.__set_uncompressed_page_size(std::numeric_limits::max()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(encoding); + EXPECT_TRUE(validate_fixed_width_page_size(header, sizeof(int32_t), 0, 0) + .is()); + header.__set_uncompressed_page_size(sizeof(int32_t)); + EXPECT_TRUE(validate_fixed_width_page_size(header, sizeof(int32_t), 0, 0).ok()); + } +} + +TEST(ParquetV2NativeDecoderTest, OptionalV2FixedWidthPageRejectsExtentBeforeAllocation) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::array value {}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + const std::vector levels {2, 1}; + + for (const auto encoding : {tparquet::Encoding::PLAIN, tparquet::Encoding::BYTE_STREAM_SPLIT}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(levels.size() + compressed.size()); + header.__set_uncompressed_page_size((8 << 20) + levels.size()); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(1); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(encoding); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(levels.size()); + header.data_page_header_v2.__set_is_compressed(true); + std::vector payload = levels; + payload.insert(payload.end(), compressed.data(), compressed.data() + compressed.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, 1, nullptr, + context); + ASSERT_TRUE(reader.init().ok()); + EXPECT_TRUE(reader.load_page_data().is()); + EXPECT_LT(reader.retained_decoder_scratch_bytes(), 64UL << 10); + } +} + +TEST(ParquetV2NativeDecoderTest, RepeatedV2FixedWidthPageRejectsExtentBeforeAllocation) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::array value {}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + const std::vector levels {2, 0, 2, 1}; + + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(levels.size() + compressed.size()); + header.__set_uncompressed_page_size((8 << 20) + levels.size()); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(1); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(2); + header.data_page_header_v2.__set_definition_levels_byte_length(2); + header.data_page_header_v2.__set_is_compressed(true); + std::vector payload = levels; + payload.insert(payload.end(), compressed.data(), compressed.data() + compressed.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::INT32); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, 1, nullptr, context); + ASSERT_TRUE(reader.init().ok()); + EXPECT_TRUE(reader.load_page_data().is()); + EXPECT_LT(reader.retained_decoder_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, VariableWidthDataPagePreflightsCompressedExtent) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::vector value {1, 0, 0, 0, 'x'}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(8 << 20); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + const std::vector payload(compressed.data(), compressed.data() + compressed.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader stream(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::BYTE_ARRAY); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::SNAPPY); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::BYTE_ARRAY; + field.definition_level = 1; + field.parquet_schema.__set_type(tparquet::Type::BYTE_ARRAY); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext context(false, ""); + ColumnChunkReader reader(&stream, &chunk, &field, nullptr, 1, nullptr, context); + ASSERT_TRUE(reader.init().ok()); + EXPECT_TRUE(reader.load_page_data().is()); + EXPECT_LT(reader.retained_decoder_scratch_bytes(), 64UL << 10); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY, true, + tparquet::Type::BYTE_ARRAY) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, VariableWidthDictionaryPreflightsCompressedExtent) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + const std::vector value {1, 0, 0, 0, 'x'}; + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(value.data(), value.size()), &compressed).ok()); + const Slice payload(compressed.data(), compressed.size()); + + EXPECT_TRUE( + validate_compressed_page_size(tparquet::CompressionCodec::SNAPPY, payload, value.size()) + .ok()); + EXPECT_TRUE(validate_compressed_page_size(tparquet::CompressionCodec::SNAPPY, payload, 8 << 20) + .is()); + + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(compressed.size()); + header.__set_uncompressed_page_size(8 << 20); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE( + load_scripted_page( + header, + std::vector(compressed.data(), compressed.data() + compressed.size()), + tparquet::CompressionCodec::SNAPPY, false, tparquet::Type::BYTE_ARRAY) + .is()); + EXPECT_TRUE( + load_scripted_page( + header, + std::vector(compressed.data(), compressed.data() + compressed.size()), + tparquet::CompressionCodec::SNAPPY, true, tparquet::Type::BYTE_ARRAY) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, UncompressedDataPagesRequireEqualPhysicalAndLogicalSizes) { + for (const auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + } + const std::vector payload(8); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED) + .is()); + EXPECT_TRUE( + load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED, true) + .is()); + } +} + +TEST(ParquetV2NativeDecoderTest, LegacyV2CompressedOverrideAllowsDifferentSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(16); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, true) + .ok()); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, EmptyOffsetIndexCannotSelectIndexedPageReader) { + MemoryBufferedReader reader(std::vector {0}); + tparquet::ColumnMetaData metadata; + tparquet::OffsetIndex empty_index; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, 1, 1, metadata, context, &empty_index); + EXPECT_FALSE(page_reader.has_next_page()); + EXPECT_TRUE(page_reader.next_page().is()); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkRangeRejectsSignedOverflowAndBoundsLegacyPadding) { + tparquet::ColumnMetaData metadata; + metadata.__set_data_page_offset(-1); + metadata.__set_total_compressed_size(10); + ColumnChunkRange range; + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(95); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(10); + metadata.__set_total_compressed_size(std::numeric_limits::max()); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_total_compressed_size(20); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, true, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 25); + + metadata.__set_dictionary_page_offset(0); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 20); + + metadata.__set_dictionary_page_offset(5); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 5); + EXPECT_EQ(range.length, 20); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsBackwardAndOverlappingLocations) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(110); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(120); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); + + second.__set_offset(140); + second.__set_first_row_index(0); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); + + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_TRUE(validate_offset_index(index, range, 110, 20)); + + range = {.offset = std::numeric_limits::max(), .length = 2}; + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsShiftedFirstDataPage) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(120); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(140); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + + EXPECT_FALSE(validate_offset_index(index, range, 100, 20)); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingDataDecoder) { + tparquet::PageHeader index_header; + index_header.type = tparquet::PageType::INDEX_PAGE; + index_header.__set_compressed_page_size(3); + index_header.__set_uncompressed_page_size(3); + index_header.__set_index_page_header(tparquet::IndexPageHeader()); + auto bytes = serialize_page(index_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsUnknownAuxiliaryPage) { + tparquet::PageHeader unknown_header; + unknown_header.type = static_cast(127); + unknown_header.__set_compressed_page_size(3); + unknown_header.__set_uncompressed_page_size(3); + auto bytes = serialize_page(unknown_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); +} + +TEST(ParquetV2NativeDecoderTest, LegacyDataPageV2OverridesFalseCompressedFlag) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(1024); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + EXPECT_TRUE(should_cache_decompressed(&header, metadata, false)); + EXPECT_FALSE(should_cache_decompressed(&header, metadata, true)); + + EXPECT_TRUE(parquet_reader_compat("parquet-cpp version 1.5.0").data_page_v2_always_compressed); + EXPECT_FALSE(parquet_reader_compat("parquet-cpp version 2.0.0").data_page_v2_always_compressed); + EXPECT_TRUE(parquet_reader_compat("parquet-mr version 1.2.8").parquet_816_padding); + EXPECT_FALSE(parquet_reader_compat("parquet-mr version 1.2.9").parquet_816_padding); +} + +TEST(ParquetV2NativeDecoderTest, NullableNumericOverflowIsNullOnlyOutsideStrictMode) { + const int32_t overflow = 1000; + auto decode = [&](bool strict, MutableColumnPtr* column, IColumn::Filter* null_map) { + std::unique_ptr decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder)); + decoder->set_type_length(sizeof(overflow)); + Slice slice(reinterpret_cast(&overflow), sizeof(overflow)); + RETURN_IF_ERROR(decoder->set_data(&slice)); + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + MutableColumnPtr column; + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ASSERT_TRUE(decode(false, &column, &null_map).ok()); + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + + null_map.clear(); + null_map.resize_fill(1, 0); + EXPECT_FALSE(decode(true, &column, &null_map).ok()); + EXPECT_EQ(null_map[0], 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryConversionFailureIsDeferredUntilReferenced) { + auto decode_dictionary_id = [](uint8_t dictionary_id, bool strict, IColumn::Filter* null_map, + MutableColumnPtr* column) { + const std::array dictionary_values {1, 1000}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + RETURN_IF_ERROR(Decoder::get_decoder(tparquet::Type::INT32, + tparquet::Encoding::RLE_DICTIONARY, decoder)); + decoder->set_type_length(sizeof(int32_t)); + RETURN_IF_ERROR( + decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size())); + char encoded_id[] = {1, 2, static_cast(dictionary_id)}; + Slice id_slice(encoded_id, sizeof(encoded_id)); + RETURN_IF_ERROR(decoder->set_data(&id_slice)); + + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + for (const bool strict : {false, true}) { + IColumn::Filter null_map; + IColumn::Filter* output_null_map = nullptr; + if (strict) { + null_map.resize_fill(1, 0); + output_null_map = &null_map; + } + + MutableColumnPtr column; + const auto unused_bad = decode_dictionary_id(0, strict, output_null_map, &column); + ASSERT_TRUE(unused_bad.ok()) << unused_bad; + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(assert_cast(*column).get_element(0), 1); + + const auto referenced_bad = decode_dictionary_id(1, strict, output_null_map, &column); + EXPECT_TRUE(referenced_bad.is()) << referenced_bad; + EXPECT_TRUE(column->empty()); + if (output_null_map != nullptr) { + EXPECT_EQ((*output_null_map)[0], 0); + } + } +} + +TEST(ParquetV2NativeDecoderTest, FixedLengthStringsAppendAsOneContiguousSpan) { + ColumnString column; + const std::string values = "aaabbbccc"; + column.insert_many_fixed_length_data(values.data(), 3, 3); + ASSERT_EQ(column.size(), 3); + EXPECT_EQ(column.get_data_at(0).to_string_view(), "aaa"); + EXPECT_EQ(column.get_data_at(1).to_string_view(), "bbb"); + EXPECT_EQ(column.get_data_at(2).to_string_view(), "ccc"); +} + +TEST(ParquetV2NativeDecoderTest, ComplexPageStatisticsPreservePerLeafCrossings) { + ColumnChunkReaderStatistics first_chunk; + first_chunk.page_read_counter = 1; + first_chunk.data_page_read_counter = 1; + ColumnChunkReaderStatistics second_chunk; + second_chunk.page_read_counter = 1; + second_chunk.data_page_read_counter = 1; + ColumnReader::ColumnStatistics combined; + ColumnReader::ColumnStatistics first(first_chunk, 0); + ColumnReader::ColumnStatistics second(second_chunk, 0); + combined.merge(first); + combined.merge(second); + + EXPECT_EQ(combined.page_read_counter, 2); + ASSERT_EQ(combined.leaf_page_read_counters.size(), 2); + EXPECT_EQ(combined.leaf_page_read_counters[0], 1); + EXPECT_EQ(combined.leaf_page_read_counters[1], 1); +} + +TEST(ParquetV2NativeDecoderTest, NativePageCacheUsesStableFileDescriptionIdentity) { + ParquetPageCacheKeyBuilder first; + ParquetPageCacheKeyBuilder replaced; + first.init("s3://bucket/object::etag-v1::1234"); + replaced.init("s3://bucket/object::etag-v2::1234"); + EXPECT_EQ(first.make_key(4096, 128).fname, "s3://bucket/object::etag-v1::1234"); + EXPECT_NE(first.make_key(4096, 128).encode(), replaced.make_key(4096, 128).encode()); +} + +TEST(ParquetV2NativeDecoderTest, UncompressedV2PageCachePayloadIsAlwaysDecompressed) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(100); + header.__set_uncompressed_page_size(1000); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + + // The representation is explicit in V2; a codec and compression ratio cannot override it on + // a warm cache hit. + EXPECT_TRUE(should_cache_decompressed(&header, metadata)); +} + +TEST(ParquetV2NativeDecoderTest, CacheDisabledPagesDoNotPrepareCopiedPayload) { + EXPECT_FALSE(can_prepare_page_cache_payload(false, false, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, true, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, false, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, true, false)); + EXPECT_TRUE(can_prepare_page_cache_payload(true, false, true, true)); +} + +TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchUsesIdleBatchHysteresis) { + ::doris::RowRanges row_ranges; + tparquet::ColumnChunk chunk; + ScalarColumnReader reader(row_ranges, 1, chunk, nullptr, nullptr, nullptr); + + constexpr size_t max_retained_bytes = 64UL << 10; + reader.reserve_batch_scratch_for_test(1UL << 16); + const size_t oversized_bytes = reader.retained_batch_scratch_bytes_for_test(); + ASSERT_GT(oversized_bytes, max_retained_bytes); + + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); + reader.release_batch_scratch(max_retained_bytes); + const size_t released_bytes = reader.retained_batch_scratch_bytes_for_test(); + EXPECT_LT(released_bytes, oversized_bytes); + EXPECT_LE(released_bytes, max_retained_bytes + sizeof(void*)); +} + +} // namespace +} // namespace doris::format::parquet::native diff --git a/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp new file mode 100644 index 00000000000000..490a0bde204a74 --- /dev/null +++ b/be/test/format_v2/parquet/parquet_benchmark_scenarios_test.cpp @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "../../../benchmark/parquet/parquet_benchmark_scenarios.h" + +#include + +#include +#include +#include +#include + +namespace doris::parquet_benchmark { +namespace { + +TEST(ParquetBenchmarkScenariosTest, DecoderMatrixCoversNativeEncodingAndTypeFamilies) { + const auto scenarios = decoder_scenarios(); + const std::set> actual = [&] { + std::set> values; + for (const auto& scenario : scenarios) { + values.emplace(scenario.encoding, scenario.value_type); + } + return values; + }(); + + const std::set> expected { + {Encoding::PLAIN, ValueType::INT32}, + {Encoding::PLAIN, ValueType::INT64}, + {Encoding::PLAIN, ValueType::FLOAT}, + {Encoding::PLAIN, ValueType::DOUBLE}, + {Encoding::PLAIN, ValueType::BYTE_ARRAY}, + {Encoding::PLAIN, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::INT32}, + {Encoding::DICTIONARY, ValueType::INT64}, + {Encoding::DICTIONARY, ValueType::FLOAT}, + {Encoding::DICTIONARY, ValueType::DOUBLE}, + {Encoding::DICTIONARY, ValueType::BYTE_ARRAY}, + {Encoding::DICTIONARY, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FLOAT}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::DOUBLE}, + {Encoding::BYTE_STREAM_SPLIT, ValueType::FIXED_LEN_BYTE_ARRAY}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT32}, + {Encoding::DELTA_BINARY_PACKED, ValueType::INT64}, + {Encoding::DELTA_LENGTH_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + {Encoding::DELTA_BYTE_ARRAY, ValueType::BYTE_ARRAY}, + }; + EXPECT_EQ(actual, expected); +} + +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversNullableSparseAndProjectionAxes) { + const auto scenarios = reader_scenarios(); + for (const int null_percent : {0, 1, 10, 50, 90}) { + for (const auto pattern : {Pattern::CLUSTERED, Pattern::ALTERNATING}) { + for (const int selectivity : {0, 1, 10, 50, 90, 100}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.encoding == Encoding::PLAIN && + scenario.null_percent == null_percent && + scenario.null_pattern == pattern && + scenario.selectivity_percent == selectivity && + scenario.projection == projection && scenario.schema_width == 32; + })) << "missing nullable sparse axis combination"; + } + } + } + } +} + +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversOperationsEncodingsAndSchemaWidths) { + const auto scenarios = reader_scenarios(); + for (const auto operation : + {ReaderOperation::OPEN_TO_FIRST_BLOCK, ReaderOperation::FULL_SCAN, + ReaderOperation::PREDICATE_SCAN, ReaderOperation::LIMIT_1, ReaderOperation::LIMIT_1000}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == operation; + })); + } + for (const auto encoding : {Encoding::PLAIN, Encoding::DICTIONARY, Encoding::BYTE_STREAM_SPLIT, + Encoding::DELTA_BINARY_PACKED}) { + for (const auto operation : {ReaderOperation::FULL_SCAN, ReaderOperation::PREDICATE_SCAN}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.encoding == encoding && scenario.operation == operation; + })); + } + } + for (const int width : {4, 32, 128, 512}) { + for (const int predicate_position : {0, width - 1}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.schema_width == width && + scenario.predicate_position == predicate_position; + })); + } + } +} + +TEST(ParquetBenchmarkScenariosTest, ReaderMatrixCoversFixedWidthRawFilterAxes) { + const auto scenarios = reader_scenarios(); + for (const auto encoding : {Encoding::BYTE_STREAM_SPLIT, Encoding::DELTA_BINARY_PACKED}) { + for (const int selectivity : {1, 10, 50, 90}) { + for (const auto projection : + {Projection::PREDICATE_ONLY, Projection::PREDICATE_PROJECTED}) { + EXPECT_TRUE(std::ranges::any_of(scenarios, [&](const ReaderScenario& scenario) { + return scenario.operation == ReaderOperation::PREDICATE_SCAN && + scenario.encoding == encoding && scenario.null_percent == 10 && + scenario.null_pattern == Pattern::ALTERNATING && + scenario.selectivity_percent == selectivity && + scenario.projection == projection && scenario.schema_width == 32 && + scenario.predicate_position == 0; + })) << "missing fixed-width raw filter axis combination"; + } + } + } +} + +TEST(ParquetBenchmarkScenariosTest, SelectionPlanDistinguishesClusteredAndSparseRuns) { + const auto clustered = make_selection_plan(1000, 10, Pattern::CLUSTERED); + EXPECT_EQ(clustered.total_rows, 1000); + EXPECT_EQ(clustered.selected_rows, 100); + ASSERT_EQ(clustered.ranges.size(), 1); + EXPECT_EQ(clustered.ranges.front().count, 100); + + const auto alternating = make_selection_plan(1000, 10, Pattern::ALTERNATING); + EXPECT_EQ(alternating.total_rows, 1000); + EXPECT_EQ(alternating.selected_rows, 100); + EXPECT_EQ(alternating.ranges.size(), 100); + + EXPECT_EQ(make_selection_plan(1000, 0, Pattern::ALTERNATING).ranges.size(), 0); + EXPECT_EQ(make_selection_plan(1000, 100, Pattern::ALTERNATING).ranges.size(), 1); +} + +} // namespace +} // namespace doris::parquet_benchmark diff --git a/be/test/format_v2/parquet/parquet_column_reader_test.cpp b/be/test/format_v2/parquet/parquet_column_reader_test.cpp deleted file mode 100644 index fb4cd129e1b03d..00000000000000 --- a/be/test/format_v2/parquet/parquet_column_reader_test.cpp +++ /dev/null @@ -1,3682 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_decimal.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_struct.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "core/types.h" -#include "format_v2/file_reader.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/selection_vector.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -template -const ColumnType& get_nullable_nested_column(const IColumn& column) { - // File-local schema exposed by the parquet reader follows Doris external-table semantics: - // nested STRUCT fields, LIST elements, and MAP keys/values are nullable even when the parquet - // field is required. - const auto& nullable_column = assert_cast(column); - return assert_cast(nullable_column.get_nested_column()); -} - -ParquetColumnSchema mock_column_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "mock"; - schema.type = std::make_shared(); - return schema; -} - -class BaseUnsupportedReader final : public ParquetColumnReader { -public: - BaseUnsupportedReader() - : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } -}; - -class DefaultSelectReader final : public ParquetColumnReader { -public: - DefaultSelectReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - auto& values = assert_cast(*column); - for (int64_t row = 0; row < rows; ++row) { - values.insert_value(static_cast(_cursor + row)); - } - _cursor += rows; - *rows_read = rows; - _read_ranges.push_back(rows); - return Status::OK(); - } - - Status skip(int64_t rows) override { - _cursor += rows; - _skip_ranges.push_back(rows); - return Status::OK(); - } - - const std::vector& read_ranges() const { return _read_ranges; } - const std::vector& skip_ranges() const { return _skip_ranges; } - -private: - int64_t _cursor = 0; - std::vector _read_ranges; - std::vector _skip_ranges; -}; - -class NestedSkipReader final : public ParquetColumnReader { -public: - NestedSkipReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - *values_consumed = length_upper_bound; - return Status::OK(); - } -}; - -class ParquetColumnReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_column_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "reader.parquet").string(); - _plain_file_path = (_test_dir / "plain_reader.parquet").string(); - write_parquet_file(); - _file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto metadata = _file_reader->metadata(); - ASSERT_EQ(metadata->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(_fields.size(), _expected_by_field.size()); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_all_null_int32_array() { - arrow::Int32Builder builder; - for (int64_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(builder.AppendNull().ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_required_struct_array() { - auto struct_type = arrow::struct_({arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - const std::vector a_values = {101, 102, 103, 104, 105}; - const std::vector b_values = {"sa", "sb", "sc", "sd", "se"}; - for (size_t row = 0; row < a_values.size(); ++row) { - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(a_values[row]).ok()); - EXPECT_TRUE(b_builder->Append(b_values[row]).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(201).ok()); - EXPECT_TRUE(b_builder->Append("nsa").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(203).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(204).ok()); - EXPECT_TRUE(b_builder->Append("nsd").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_decimal_array() { - auto decimal_type = arrow::decimal128(38, 6); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("d", decimal_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto d_array_builder = std::make_unique( - decimal_type, arrow::default_memory_pool()); - field_builders.push_back(std::shared_ptr(std::move(d_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* d_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(123456789)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(d_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(-987654321)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("xs", list_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - field_builders.push_back(list_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(305).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_map_array() { - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("kv", map_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - field_builders.push_back(map_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(401).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("one").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(403).ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(404).ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(405).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->Append("five").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_nested_struct_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto nested_type = arrow::struct_({arrow::field("xs", list_type, true)}); - auto struct_type = arrow::struct_({arrow::field("nested", nested_type, true)}); - - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - std::vector> nested_field_builders; - nested_field_builders.push_back(list_builder); - auto nested_builder = std::make_shared( - nested_type, arrow::default_memory_pool(), std::move(nested_field_builders)); - std::vector> field_builders; - field_builders.push_back(nested_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(7).ok()); - EXPECT_TRUE(value_builder->Append(8).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), false))); - const std::vector> values = { - {1, 2}, {3}, {4, 5, 6}, {7}, {8, 9}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto value : row) { - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append(20).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(30).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(110).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(120).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(130).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_list_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - auto* a_builder = assert_cast(struct_builder->field_builder(0)); - auto* b_builder = assert_cast(struct_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(11).ok()); - EXPECT_TRUE(b_builder->Append("la").ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(12).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(13).ok()); - EXPECT_TRUE(b_builder->Append("ld").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(14).ok()); - EXPECT_TRUE(b_builder->Append("le").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_list_list_int_array() { - auto value_builder = std::make_shared(); - auto inner_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto inner_list_builder = std::make_shared( - arrow::default_memory_pool(), value_builder, inner_list_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), inner_list_builder, - arrow::list(arrow::field("element", inner_list_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->AppendNull().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - const std::vector>> values = { - {{1, "a"}, {2, "b"}}, {{3, "c"}}, {{4, "d"}, {5, "e"}, {6, "f"}}, - {{7, "g"}}, {{8, "h"}, {9, "i"}}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto& [key, value] : row) { - EXPECT_TRUE(key_builder->Append(key).ok()); - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append("bb").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_struct_map_array() { - auto key_builder = std::make_shared(); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto value_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", struct_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - auto* a_builder = assert_cast(value_builder->field_builder(0)); - auto* b_builder = assert_cast(value_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(21).ok()); - EXPECT_TRUE(b_builder->Append("ma").ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(22).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(24).ok()); - EXPECT_TRUE(b_builder->Append("me").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(201).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(key_builder->Append(202).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(203).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(204).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(205).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_map_list_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), map_builder, - arrow::list(arrow::field("element", map_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("a").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(value_builder->Append("d").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_map_map_array() { - auto key_builder = std::make_shared(); - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, nested_map_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(101).ok()); - EXPECT_TRUE(nested_value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(401).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_list_struct_map_list_array() { - auto element_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - element_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - auto map_builder = std::make_shared(arrow::default_memory_pool(), - key_builder, list_builder, map_type); - auto struct_type = arrow::struct_({arrow::field("kv", map_type, true)}); - std::vector> struct_field_builders; - struct_field_builders.push_back(map_builder); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(struct_field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(10).ok()); - EXPECT_TRUE(element_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_map_list_map_array() { - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto list_type = arrow::list(arrow::field("element", nested_map_type, true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - nested_map_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(1).ok()); - EXPECT_TRUE(nested_value_builder->Append("a").ok()); - EXPECT_TRUE(nested_key_builder->Append(2).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(3).ok()); - EXPECT_TRUE(nested_value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(50).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(4).ok()); - EXPECT_TRUE(nested_value_builder->Append("d").ok()); - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, std::shared_ptr array, - std::function validator) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - _expected_by_field.push_back(std::move(validator)); - } - - void write_parquet_file() { - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); - add_field(arrow::field("all_null_int_col", arrow::int32(), true), - build_all_null_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - for (size_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(nullable_column.is_null_at(row)); - } - }); - add_field(arrow::field("struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false), - }), - false), - build_required_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_STRUCT); - const auto& struct_column = assert_cast(column); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - get_nullable_nested_column(struct_column.get_column(1)); - EXPECT_EQ(a_values.get_element(0), 101); - EXPECT_EQ(a_values.get_element(4), 105); - EXPECT_EQ(b_values.get_data_at(1).to_string(), "sb"); - EXPECT_EQ(b_values.get_data_at(4).to_string(), "se"); - }); - add_field(arrow::field("nullable_struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }), - true), - build_nullable_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_nested = - assert_cast(b_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 201); - EXPECT_EQ(a_values.get_element(2), 203); - EXPECT_EQ(a_values.get_element(3), 204); - EXPECT_FALSE(b_values.is_null_at(0)); - EXPECT_TRUE(b_values.is_null_at(2)); - EXPECT_FALSE(b_values.is_null_at(3)); - EXPECT_EQ(b_nested.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(b_nested.get_data_at(3).to_string(), "nsd"); - }); - add_field(arrow::field("nullable_struct_decimal_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("d", arrow::decimal128(38, 6), true), - }), - true), - build_nullable_struct_with_decimal_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& d_values = - assert_cast(struct_column.get_column(1)); - const auto& d_nested = - assert_cast(d_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_FALSE(d_values.is_null_at(0)); - EXPECT_TRUE(d_values.is_null_at(2)); - EXPECT_FALSE(d_values.is_null_at(3)); - EXPECT_EQ(d_nested.get_element(0), Decimal128V3(123456789)); - EXPECT_EQ(d_nested.get_element(3), Decimal128V3(-987654321)); - }); - auto struct_list_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("xs", arrow::list(arrow::field("element", arrow::int32(), true)), - true), - }); - add_field(arrow::field("nullable_struct_list_col", struct_list_type, true), - build_nullable_struct_with_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_EQ(a_values.get_element(4), 305); - - const auto& xs_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = - assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = - assert_cast(xs_array.get_data()); - ASSERT_EQ(elements.size(), 4); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - const auto& values = - assert_cast(elements.get_nested_column()); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_EQ(values.get_element(3), 5); - }); - auto struct_map_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("kv", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - }); - add_field(arrow::field("nullable_struct_map_col", struct_map_type, true), - build_nullable_struct_with_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(2), 403); - EXPECT_EQ(a_values.get_element(3), 404); - EXPECT_EQ(a_values.get_element(4), 405); - - const auto& kv_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = - assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); - }); - auto nested_struct_list_type = arrow::struct_({ - arrow::field("nested", - arrow::struct_({ - arrow::field("xs", - arrow::list(arrow::field("element", - arrow::int32(), true)), - true), - }), - true), - }); - add_field(arrow::field("nullable_struct_nested_struct_list_col", nested_struct_list_type, - true), - build_nullable_struct_with_nested_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = - assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); - }); - add_field(arrow::field("list_int_col", - arrow::list(arrow::field("element", arrow::int32(), false)), false), - build_required_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_ARRAY); - const auto* array_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ( - remove_nullable(array_type->get_nested_type())->get_primitive_type(), - TYPE_INT); - const auto& array_column = assert_cast(column); - ASSERT_EQ(array_column.size(), ROW_COUNT); - const auto array_size_at = [&array_column](size_t row_idx) { - return array_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : array_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(array_size_at(0), 2); - EXPECT_EQ(array_size_at(1), 1); - EXPECT_EQ(array_size_at(2), 3); - EXPECT_EQ(array_size_at(4), 2); - const auto& values = - get_nullable_nested_column(array_column.get_data()); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(5), 6); - EXPECT_EQ(values.get_element(8), 9); - }); - add_field(arrow::field("nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), true), - build_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 30); - EXPECT_EQ(values.get_element(4), 40); - }); - add_field(arrow::field("required_nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), false), - build_required_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& array_column = assert_cast(column); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 5); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - ASSERT_EQ(elements.size(), 5); - EXPECT_TRUE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(4)); - }); - auto list_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field("nullable_list_struct_col", - arrow::list(arrow::field("element", list_struct_type, true)), true), - build_nullable_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - - const auto& elements = - assert_cast(array_column.get_data()); - const auto& struct_column = - assert_cast(elements.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 11); - EXPECT_EQ(a_values.get_element(1), 12); - EXPECT_EQ(a_values.get_element(3), 13); - EXPECT_EQ(a_values.get_element(4), 14); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); - }); - auto nested_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field(arrow::field("nullable_list_list_int_col", - arrow::list(arrow::field("element", nested_list_type, true)), true), - build_nullable_list_list_int_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 4); - EXPECT_EQ(outer_offsets[3], 5); - EXPECT_EQ(outer_offsets[4], 7); - - const auto& inner_nullable = - assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_FALSE(inner_nullable.is_null_at(6)); - - const auto& inner_array = - assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[5], 5); - EXPECT_EQ(inner_offsets[6], 7); - - const auto& elements = - assert_cast(inner_array.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 7); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 3); - EXPECT_EQ(values.get_element(4), 4); - EXPECT_EQ(values.get_element(5), 5); - EXPECT_TRUE(elements.is_null_at(6)); - }); - add_field(arrow::field( - "map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)), - false), - build_required_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_MAP); - const auto* map_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), - TYPE_INT); - EXPECT_EQ(remove_nullable(map_type->get_value_type())->get_primitive_type(), - TYPE_STRING); - const auto& map_column = assert_cast(column); - ASSERT_EQ(map_column.size(), ROW_COUNT); - const auto map_size_at = [&map_column](size_t row_idx) { - return map_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : map_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(map_size_at(0), 2); - EXPECT_EQ(map_size_at(1), 1); - EXPECT_EQ(map_size_at(2), 3); - EXPECT_EQ(map_size_at(4), 2); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(keys.size(), 9); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(5), 6); - EXPECT_EQ(keys.get_element(8), 9); - EXPECT_EQ(values.get_data_at(0).to_string(), "a"); - EXPECT_EQ(values.get_data_at(5).to_string(), "f"); - EXPECT_EQ(values.get_data_at(8).to_string(), "i"); - }); - add_field( - arrow::field("nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - build_nullable_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 10); - EXPECT_EQ(keys.get_element(1), 20); - EXPECT_EQ(keys.get_element(3), 40); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(values.is_null_at(3)); - }); - add_field( - arrow::field("required_nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - false), - build_required_nullable_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& map_column = assert_cast(column); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 4); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(3)); - }); - auto map_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field( - "nullable_map_int_struct_col", - arrow::map(arrow::int32(), arrow::field("value", map_struct_type, true)), - true), - build_nullable_int_struct_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& struct_column = - assert_cast(values.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 21); - EXPECT_EQ(a_values.get_element(1), 22); - EXPECT_EQ(a_values.get_element(3), 24); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); - }); - auto map_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field( - arrow::field("nullable_map_int_list_col", - arrow::map(arrow::int32(), arrow::field("value", map_list_type, true)), - true), - build_nullable_int_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), ROW_COUNT); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 4); - EXPECT_EQ(map_offsets[4], 5); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 5); - EXPECT_EQ(keys.get_element(0), 201); - EXPECT_EQ(keys.get_element(1), 202); - EXPECT_EQ(keys.get_element(2), 203); - EXPECT_EQ(keys.get_element(3), 204); - EXPECT_EQ(keys.get_element(4), 205); - - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_FALSE(values.is_null_at(4)); - - const auto& list_column = - assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); - - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(element_values.get_element(0), 1); - EXPECT_EQ(element_values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(element_values.get_element(3), 3); - EXPECT_EQ(element_values.get_element(4), 4); - }); - auto list_map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field("nullable_list_map_int_string_col", - arrow::list(arrow::field("element", list_map_type, true)), true), - build_nullable_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& map_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 3); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(value_data.get_data_at(3).to_string(), "d"); - }); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field( - "nullable_map_int_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)), - true), - build_nullable_int_map_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 4); - - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 4); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - - const auto& inner_values = - assert_cast(outer_map.get_values()); - ASSERT_EQ(inner_values.size(), 4); - EXPECT_FALSE(inner_values.is_null_at(0)); - EXPECT_FALSE(inner_values.is_null_at(1)); - EXPECT_TRUE(inner_values.is_null_at(2)); - EXPECT_FALSE(inner_values.is_null_at(3)); - - const auto& inner_map = - assert_cast(inner_values.get_nested_column()); - const auto& inner_offsets = inner_map.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 4); - EXPECT_EQ(inner_offsets[0], 1); - EXPECT_EQ(inner_offsets[1], 1); - EXPECT_EQ(inner_offsets[2], 1); - EXPECT_EQ(inner_offsets[3], 2); - const auto& inner_keys = - get_nullable_nested_column(inner_map.get_keys()); - const auto& inner_strings = - assert_cast(inner_map.get_values()); - const auto& inner_string_data = - assert_cast(inner_strings.get_nested_column()); - ASSERT_EQ(inner_keys.size(), 2); - EXPECT_EQ(inner_keys.get_element(0), 101); - EXPECT_EQ(inner_keys.get_element(1), 401); - EXPECT_EQ(inner_string_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(inner_strings.is_null_at(1)); - }); - auto deep_list_value_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto deep_list_map_type = - arrow::map(arrow::int32(), arrow::field("value", deep_list_value_type, true)); - auto deep_list_struct_type = arrow::struct_({arrow::field("kv", deep_list_map_type, true)}); - add_field(arrow::field("nullable_list_struct_map_list_col", - arrow::list(arrow::field("element", deep_list_struct_type, true)), - true), - build_deep_list_struct_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& struct_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - - const auto& struct_column = - assert_cast(struct_values.get_nested_column()); - const auto& map_values = - assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - - const auto& lists = - assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 4); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 4); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 3); - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 3); - EXPECT_EQ(element_values.get_element(0), 10); - EXPECT_TRUE(elements.is_null_at(1)); - EXPECT_EQ(element_values.get_element(2), 40); - }); - auto deep_map_nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto deep_map_list_type = - arrow::list(arrow::field("element", deep_map_nested_map_type, true)); - add_field( - arrow::field( - "nullable_map_int_list_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", deep_map_list_type, true)), - true), - build_deep_map_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); - - const auto& inner_maps = - assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 6); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_FALSE(inner_maps.is_null_at(1)); - EXPECT_TRUE(inner_maps.is_null_at(2)); - EXPECT_FALSE(inner_maps.is_null_at(3)); - EXPECT_TRUE(inner_maps.is_null_at(4)); - EXPECT_FALSE(inner_maps.is_null_at(5)); - const auto& inner_map_column = - assert_cast(inner_maps.get_nested_column()); - const auto& inner_offsets = inner_map_column.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 6); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 3); - EXPECT_EQ(inner_offsets[4], 3); - EXPECT_EQ(inner_offsets[5], 4); - const auto& inner_keys = - get_nullable_nested_column(inner_map_column.get_keys()); - ASSERT_EQ(inner_keys.size(), 4); - EXPECT_EQ(inner_keys.get_element(0), 1); - EXPECT_EQ(inner_keys.get_element(1), 2); - EXPECT_EQ(inner_keys.get_element(2), 3); - EXPECT_EQ(inner_keys.get_element(3), 4); - const auto& strings = - assert_cast(inner_map_column.get_values()); - const auto& string_data = - assert_cast(strings.get_nested_column()); - ASSERT_EQ(strings.size(), 4); - EXPECT_EQ(string_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(strings.is_null_at(1)); - EXPECT_EQ(string_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(string_data.get_data_at(3).to_string(), "d"); - }); - - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - - auto file_result = arrow::io::FileOutputStream::Open(_file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ROW_COUNT, builder.build())); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_plain_reader(size_t field_idx) { - // Keep the normal fixture dictionary encoded. This one test writes a plain-encoded copy - // because Arrow BinaryRecordReader has a stricter reset contract than - // DictionaryRecordReader. - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - auto plain_file_result = arrow::io::FileOutputStream::Open(_plain_file_path); - DORIS_CHECK(plain_file_result.ok()); - std::shared_ptr plain_out = *plain_file_result; - ::parquet::WriterProperties::Builder plain_builder; - plain_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - plain_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - plain_builder.compression(::parquet::Compression::UNCOMPRESSED); - plain_builder.disable_dictionary(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), plain_out, ROW_COUNT, plain_builder.build())); - DORIS_CHECK(plain_out->Close().ok()); - - _plain_file_reader = ::parquet::ParquetFileReader::OpenFile(_plain_file_path, false); - auto metadata = _plain_file_reader->metadata(); - DORIS_CHECK(metadata != nullptr); - DORIS_CHECK(metadata->num_row_groups() == 1); - _plain_row_group = _plain_file_reader->RowGroup(0); - DORIS_CHECK(_plain_row_group != nullptr); - - ParquetColumnReaderFactory factory(_plain_row_group, metadata->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_child_reader(size_t field_idx, - size_t child_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[child_idx]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_grandchild_reader( - size_t field_idx, size_t child_idx, size_t grandchild_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - const auto& child_schema = *struct_schema.children[child_idx]; - EXPECT_LT(grandchild_idx, child_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = child_schema.local_id; - child_projection.project_all_children = false; - format::LocalColumnIndex grandchild_projection; - grandchild_projection.index = child_schema.children[grandchild_idx]->local_id; - child_projection.children.push_back(std::move(grandchild_projection)); - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - void read_and_validate(size_t field_idx) const { - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - _expected_by_field[field_idx](*_fields[field_idx], *column); - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet test field " << name; - return _fields.size(); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::string _plain_file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::unique_ptr<::parquet::ParquetFileReader> _plain_file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::shared_ptr<::parquet::RowGroupReader> _plain_row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; - std::vector> _expected_by_field; -}; - -TEST(ParquetColumnReaderBaseTest, SelectionVectorRangesAndValidation) { - SelectionVector identity; - ASSERT_TRUE(identity.verify(4, 5).ok()); - auto ranges = selection_to_ranges(identity, 4); - ASSERT_EQ(ranges.size(), 1); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 4); - - std::array selected = {0, 2, 3, 6, 6}; - SelectionVector external(selected.data(), 4); - auto status = external.verify(3, 7); - ASSERT_TRUE(status.ok()) << status; - ranges = selection_to_ranges(external, 3); - ASSERT_EQ(ranges.size(), 2); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 1); - EXPECT_EQ(ranges[1].start, 2); - EXPECT_EQ(ranges[1].length, 2); - - EXPECT_FALSE(external.verify(8, 7).ok()); - EXPECT_FALSE(external.verify(5, 7).ok()); - EXPECT_FALSE(external.verify(4, 6).ok()); - - std::array duplicate = {0, 2, 2}; - SelectionVector non_strict(duplicate.data(), duplicate.size()); - EXPECT_FALSE(non_strict.verify(3, 5).ok()); - EXPECT_FALSE(identity.verify(1, -1).ok()); -} - -TEST(ParquetColumnReaderBaseTest, DefaultSelectUsesSkipReadRangesAndNestedConsumeIsExplicit) { - DefaultSelectReader reader; - std::array selected = {1, 3, 4}; - SelectionVector selection(selected.data(), selected.size()); - auto column = ColumnInt32::create(); - MutableColumnPtr mutable_column = std::move(column); - auto status = reader.select(selection, selected.size(), 6, mutable_column); - ASSERT_TRUE(status.ok()) << status; - - const auto& values = assert_cast(*mutable_column); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 3); - EXPECT_EQ(values.get_element(2), 4); - EXPECT_EQ(reader.skip_ranges(), std::vector({1, 1, 1})); - EXPECT_EQ(reader.read_ranges(), std::vector({1, 2})); - - BaseUnsupportedReader unsupported_reader; - auto skip_status = unsupported_reader.skip(1); - EXPECT_FALSE(skip_status.ok()); - EXPECT_NE(skip_status.to_string().find("skip is not implemented"), std::string::npos); - EXPECT_FALSE(unsupported_reader.load_nested_batch(1).ok()); - int64_t values_read = 0; - EXPECT_FALSE(unsupported_reader.build_nested_column(1, mutable_column, &values_read).ok()); - EXPECT_FALSE(unsupported_reader.consume_nested_column(1, &values_read).ok()); - - NestedSkipReader nested_reader; - auto nested_status = nested_reader.consume_nested_column(3, &values_read); - ASSERT_TRUE(nested_status.ok()) << nested_status; - EXPECT_EQ(values_read, 3); -} - -TEST_F(ParquetColumnReaderTest, ScalarReadCoversRequiredNullableAllNullAndMultipleBatches) { - read_and_validate(find_field_idx("int32_col")); - read_and_validate(find_field_idx("string_col")); - read_and_validate(find_field_idx("nullable_int_col")); - read_and_validate(find_field_idx("all_null_int_col")); - - auto reader = create_reader(find_field_idx("int32_col")); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - ASSERT_TRUE(reader->read(3, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 3); - const auto& values = assert_cast(*column); - ASSERT_EQ(values.size(), ROW_COUNT); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_EQ(values.get_element(2), 30); - EXPECT_EQ(values.get_element(4), 50); -} - -TEST_F(ParquetColumnReaderTest, ScalarSkipCoversZeroSomeAllAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(0).ok()); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& first_value = assert_cast(*column); - EXPECT_EQ(first_value.get_element(0), 10); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(2).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& skipped_values = assert_cast(*column); - EXPECT_EQ(skipped_values.get_element(0), 30); - EXPECT_EQ(skipped_values.get_element(1), 40); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(ROW_COUNT).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - EXPECT_EQ(rows_read, 0); - EXPECT_EQ(column->size(), 0); - - reader = create_reader(find_field_idx("nullable_int_col")); - ASSERT_TRUE(reader->skip(1).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); -} - -TEST_F(ParquetColumnReaderTest, ScalarSelectCoversAllDisjointSingleZeroThenReadAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector all_selected(ROW_COUNT); - auto column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(all_selected, ROW_COUNT, ROW_COUNT, column).ok()); - const auto& all_values = assert_cast(*column); - ASSERT_EQ(all_values.size(), ROW_COUNT); - EXPECT_EQ(all_values.get_element(0), 10); - EXPECT_EQ(all_values.get_element(4), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array disjoint = {0, 2, 4}; - SelectionVector disjoint_selection(disjoint.data(), disjoint.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(disjoint_selection, disjoint.size(), ROW_COUNT, column).ok()); - const auto& disjoint_values = assert_cast(*column); - ASSERT_EQ(disjoint_values.size(), 3); - EXPECT_EQ(disjoint_values.get_element(0), 10); - EXPECT_EQ(disjoint_values.get_element(1), 30); - EXPECT_EQ(disjoint_values.get_element(2), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array single = {2}; - SelectionVector single_selection(single.data(), single.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(single_selection, single.size(), ROW_COUNT, column).ok()); - const auto& single_value = assert_cast(*column); - ASSERT_EQ(single_value.size(), 1); - EXPECT_EQ(single_value.get_element(0), 30); - - reader = create_reader(find_field_idx("int32_col")); - std::array first_last = {0, 4}; - SelectionVector first_last_selection(first_last.data(), first_last.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(first_last_selection, first_last.size(), ROW_COUNT, column).ok()); - const auto& first_last_values = assert_cast(*column); - ASSERT_EQ(first_last_values.size(), 2); - EXPECT_EQ(first_last_values.get_element(0), 10); - EXPECT_EQ(first_last_values.get_element(1), 50); - - reader = create_reader(find_field_idx("int32_col")); - SelectionVector empty_selection; - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(empty_selection, 0, 2, column).ok()); - ASSERT_EQ(column->size(), 0); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& after_empty_select = assert_cast(*column); - ASSERT_EQ(after_empty_select.size(), 1); - EXPECT_EQ(after_empty_select.get_element(0), 30); - - reader = create_reader(find_field_idx("nullable_int_col")); - std::array nullable_rows = {0, 1, 2}; - SelectionVector nullable_selection(nullable_rows.data(), nullable_rows.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(nullable_selection, nullable_rows.size(), ROW_COUNT, column).ok()); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidScalarInputsAndNestedScalarProjection) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& int_schema = *_fields[find_field_idx("int32_col")]; - ParquetColumnSchema invalid_leaf; - invalid_leaf.kind = ParquetColumnSchemaKind::PRIMITIVE; - invalid_leaf.name = "invalid_leaf"; - invalid_leaf.type = int_schema.type; - invalid_leaf.type_descriptor = int_schema.type_descriptor; - invalid_leaf.descriptor = int_schema.descriptor; - invalid_leaf.leaf_column_id = _file_reader->metadata()->num_columns(); - auto status = factory.create(invalid_leaf, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); - - ParquetColumnSchema null_descriptor; - null_descriptor.kind = ParquetColumnSchemaKind::PRIMITIVE; - null_descriptor.name = "null_descriptor"; - null_descriptor.type = int_schema.type; - null_descriptor.type_descriptor = int_schema.type_descriptor; - null_descriptor.leaf_column_id = int_schema.leaf_column_id; - status = factory.create(null_descriptor, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("descriptor is null"), std::string::npos); - - const auto& list_element_schema = - *_fields[find_field_idx("nullable_list_int_col")]->children[0]; - status = factory.create(list_element_schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("flat primitive columns"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex projection = - format::LocalColumnIndex::partial_local(list_schema.local_id); - format::LocalColumnIndex element_projection = - format::LocalColumnIndex::partial_local(list_element_schema.local_id); - projection.children.push_back(std::move(element_projection)); - status = factory.create(list_schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("scalar projection is invalid"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidComplexProjections) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& struct_schema = *_fields[find_field_idx("struct_col")]; - format::LocalColumnIndex struct_empty = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - auto status = factory.create(struct_schema, &struct_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); - - format::LocalColumnIndex struct_invalid = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - struct_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(struct_schema, &struct_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex list_empty = - format::LocalColumnIndex::partial_local(list_schema.local_id); - status = factory.create(list_schema, &list_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); - - const auto& map_schema = *_fields[find_field_idx("nullable_map_int_struct_col")]; - const auto& value_schema = *map_schema.children[1]; - format::LocalColumnIndex map_invalid = - format::LocalColumnIndex::partial_local(map_schema.local_id); - map_invalid.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - map_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(map_schema, &map_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadSupportedComplexTypes) { - read_and_validate(find_field_idx("struct_col")); - read_and_validate(find_field_idx("nullable_struct_col")); - read_and_validate(find_field_idx("nullable_struct_decimal_col")); - read_and_validate(find_field_idx("list_int_col")); - read_and_validate(find_field_idx("nullable_list_int_col")); - read_and_validate(find_field_idx("required_nullable_list_int_col")); - read_and_validate(find_field_idx("nullable_list_struct_col")); - read_and_validate(find_field_idx("nullable_list_list_int_col")); - read_and_validate(find_field_idx("map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_string_col")); - read_and_validate(find_field_idx("required_nullable_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_struct_col")); - read_and_validate(find_field_idx("nullable_map_int_list_col")); - read_and_validate(find_field_idx("nullable_list_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_map_int_string_col")); - read_and_validate(find_field_idx("nullable_list_struct_map_list_col")); - read_and_validate(find_field_idx("nullable_map_int_list_map_int_string_col")); -} - -TEST_F(ParquetColumnReaderTest, SkipThenRead) { - auto reader = create_reader(find_field_idx("int32_col")); - auto st = reader->skip(2); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 2); - EXPECT_EQ(int_values.get_element(0), 30); - EXPECT_EQ(int_values.get_element(1), 40); -} - -TEST_F(ParquetColumnReaderTest, SelectReadsOnlySelectedRanges) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 2); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 3); - EXPECT_EQ(int_values.get_element(0), 10); - EXPECT_EQ(int_values.get_element(1), 30); - EXPECT_EQ(int_values.get_element(2), 50); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructChildren) { - const auto field_idx = find_field_idx("struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& struct_column = assert_cast(*column); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(values.get_data_at(0).to_string(), "sa"); - EXPECT_EQ(values.get_data_at(4).to_string(), "se"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedNullableStructChildren) { - const auto field_idx = find_field_idx("nullable_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& nullable_column = assert_cast(*column); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = assert_cast(struct_column.get_column(0)); - const auto& nested_values = assert_cast(values.get_nested_column()); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(nested_values.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(nested_values.get_data_at(3).to_string(), "nsd"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedListStructElementChildren) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& list_schema = *_fields[field_idx]; - ASSERT_EQ(list_schema.name, "nullable_list_struct_col"); - ASSERT_EQ(list_schema.children.size(), 1); - const auto& element_schema = *list_schema.children[0]; - ASSERT_EQ(element_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = list_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex element_projection; - element_projection.index = element_schema.local_id; - element_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = element_schema.children[1]->local_id; - element_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(element_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(list_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* array_type = - assert_cast(remove_nullable(reader->type()).get()); - const auto* element_type = assert_cast( - remove_nullable(array_type->get_nested_type()).get()); - ASSERT_EQ(element_type->get_elements().size(), 1); - EXPECT_EQ(element_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& elements = assert_cast(array_column.get_data()); - const auto& struct_column = assert_cast(elements.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedMapStructValueChildren) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.name, "nullable_map_int_struct_col"); - ASSERT_EQ(map_schema.children.size(), 2); - const auto& value_schema = *map_schema.children[1]; - ASSERT_EQ(value_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = map_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex value_projection; - value_projection.index = value_schema.local_id; - value_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = value_schema.children[1]->local_id; - value_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(value_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* map_type = assert_cast(remove_nullable(reader->type()).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), TYPE_INT); - const auto* value_type = - assert_cast(remove_nullable(map_type->get_value_type()).get()); - ASSERT_EQ(value_type->get_elements().size(), 1); - EXPECT_EQ(value_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - const auto& values = assert_cast(map_column.get_values()); - const auto& struct_column = assert_cast(values.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); -} - -TEST_F(ParquetColumnReaderTest, AllowsMapKeyWithValueProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - const auto& value_schema = *map_schema.children[1]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - projection.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_NE(reader, nullptr); -} - -TEST_F(ParquetColumnReaderTest, RejectMapKeyOnlyProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_FALSE(st.ok()); - EXPECT_NE(st.to_string().find("contains no value"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_list_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "xs"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = assert_cast(xs_array.get_data()); - const auto& values = assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 4); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 5); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructListChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_map_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "kv"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesListChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesMapChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesNestedStructComplexChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_nested_struct_list_col"); - auto reader = create_projected_grandchild_reader(field_idx, 0, 0); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructMapChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadStructListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructListWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(1), 304); - EXPECT_EQ(a_values.get_element(2), 305); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadStructMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(1), 404); - EXPECT_EQ(a_values.get_element(2), 405); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, ReadListStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 1); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 1); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 1); - EXPECT_EQ(inner_offsets[0], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectListListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 5); - EXPECT_EQ(outer_offsets[2], 7); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[6], 7); -} - -TEST_F(ParquetColumnReaderTest, ReadMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SkipPlainBinaryMapThenReadResetsArrowBuilder) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_plain_reader(field_idx); - - // Row 0 contains two STRING values. The levels-only skip must release (and discard) those - // Arrow BinaryRecordReader builder chunks before the next normal read. If they leak into the - // next batch, ParquetLeafReader observes more values than current definition/repetition levels. - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 3); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(map_column.get_offsets()[1], 0); - EXPECT_EQ(map_column.get_offsets()[2], 1); - const auto& values = get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(values.size(), 1); - EXPECT_EQ(values.get_data_at(0).to_string(), "cc"); -} - -TEST_F(ParquetColumnReaderTest, SelectMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectMapStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 2); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectMapListWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 4); - EXPECT_EQ(map_offsets[2], 5); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(4)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepListStructMapListAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepListStructMapListThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 3); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_FALSE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 3); - EXPECT_TRUE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_FALSE(map_values.is_null_at(2)); - - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 2); - EXPECT_EQ(keys.get_element(0), 3); - EXPECT_EQ(keys.get_element(1), 4); - const auto& lists = assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 2); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepListStructMapList) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepMapListMapAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepMapListMapThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 3); - EXPECT_EQ(outer_keys.get_element(0), 30); - EXPECT_EQ(outer_keys.get_element(1), 40); - EXPECT_EQ(outer_keys.get_element(2), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 3); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_FALSE(lists.is_null_at(2)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 3); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); - EXPECT_EQ(list_offsets[2], 3); - const auto& inner_maps = assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 3); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_TRUE(inner_maps.is_null_at(1)); - EXPECT_FALSE(inner_maps.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepMapListMap) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp b/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp deleted file mode 100644 index 0d0f9a2f8567cc..00000000000000 --- a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp +++ /dev/null @@ -1,506 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" - -namespace doris::format::parquet { -namespace { - -std::shared_ptr fixed_binary_array(const std::vector& values, - int byte_width) { - auto type = arrow::fixed_size_binary(byte_width); - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetLeafReader make_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type) { - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr); -} - -struct CapturedDecodedView { - DecodedValueKind value_kind = DecodedValueKind::INT32; - DecodedTimeUnit time_unit = DecodedTimeUnit::UNKNOWN; - int64_t row_count = 0; - int decimal_precision = -1; - int decimal_scale = -1; - int fixed_length = -1; - bool timestamp_is_adjusted_to_utc = false; - bool enable_strict_mode = false; - const cctz::time_zone* timezone = nullptr; - bool null_map_is_null = true; - std::vector null_map; - std::vector fixed_values; - std::vector binary_values; - std::vector owned_binary_values; -}; - -ParquetLeafReader make_spy_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type, - CapturedDecodedView* captured, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false) { - auto appender = [captured](MutableColumnPtr&, const DecodedColumnView& view) { - captured->value_kind = view.value_kind; - captured->time_unit = view.time_unit; - captured->row_count = view.row_count; - captured->decimal_precision = view.decimal_precision; - captured->decimal_scale = view.decimal_scale; - captured->fixed_length = view.fixed_length; - captured->timestamp_is_adjusted_to_utc = view.timestamp_is_adjusted_to_utc; - captured->enable_strict_mode = view.enable_strict_mode; - captured->timezone = view.timezone; - captured->null_map_is_null = view.null_map == nullptr; - captured->null_map.clear(); - if (view.null_map != nullptr) { - captured->null_map.assign(view.null_map, view.null_map + view.row_count); - } - captured->fixed_values.clear(); - if (view.values != nullptr && view.value_kind == DecodedValueKind::INT64) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 8); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::FLOAT) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::INT32) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } - captured->binary_values.clear(); - captured->owned_binary_values.clear(); - if (view.binary_values != nullptr) { - captured->owned_binary_values.reserve(view.binary_values->size()); - for (const auto& value : *view.binary_values) { - captured->owned_binary_values.emplace_back( - value.data == nullptr ? std::string() - : std::string(value.data, value.size)); - } - captured->binary_values.reserve(captured->owned_binary_values.size()); - for (const auto& value : captured->owned_binary_values) { - captured->binary_values.emplace_back(value.data(), value.size()); - } - } - return Status::OK(); - }; - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr, {}, timezone, - enable_strict_mode, std::move(appender)); -} - -} // namespace - -struct ParquetLeafReaderTestAccess { - static ParquetLeafBatch make_fixed_batch(const std::vector& def_levels, - const std::vector& rep_levels, - const std::vector& values, - bool read_dense_for_nullable = false) { - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._consumed_level_count = static_cast(def_levels.size()); - batch._decoded_level_count = static_cast(def_levels.size()); - batch._values_written = static_cast(values.size()); - batch._def_levels = def_levels.data(); - batch._rep_levels = rep_levels.data(); - batch._fixed_values = reinterpret_cast(values.data()); - batch._read_dense_for_nullable = read_dense_for_nullable; - return batch; - } - - static Status build_nested_batch(const ParquetLeafReader& reader, - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, - int16_t value_slot_repetition_level, - ParquetNestedScalarBatch* nested_batch) { - return reader.build_nested_batch_from_leaf_batch(leaf_batch, records_read, - value_slot_definition_level, nested_batch, - value_slot_repetition_level); - } -}; - -std::shared_ptr<::parquet::ColumnDescriptor> int32_column_descriptor(int16_t max_definition_level, - int16_t max_repetition_level) { - auto node = ::parquet::schema::PrimitiveNode::Make("leaf", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32); - return std::make_shared<::parquet::ColumnDescriptor>(node, max_definition_level, - max_repetition_level); -} - -ParquetLeafReader make_nested_leaf_reader( - const std::shared_ptr<::parquet::ColumnDescriptor>& descriptor, DataTypePtr type) { - ParquetTypeDescriptor type_descriptor; - type_descriptor.physical_type = ::parquet::Type::INT32; - type_descriptor.doris_type = type; - return ParquetLeafReader(descriptor.get(), type_descriptor, std::move(type), "nested_leaf", - nullptr); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesAreSpacedBeforeSerde) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30, 50}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& nullable = assert_cast(*column); - ASSERT_EQ(nullable.size(), 5); - EXPECT_FALSE(nullable.is_null_at(0)); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_FALSE(nullable.is_null_at(2)); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_FALSE(nullable.is_null_at(4)); - const auto& nested = assert_cast(nullable.get_nested_column()); - EXPECT_EQ(nested.get_element(0), 10); - EXPECT_EQ(nested.get_element(2), 30); - EXPECT_EQ(nested.get_element(4), 50); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesRejectCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet values"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, Float16BinaryValuesAreConvertedToFloat) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - descriptor.fixed_length = 2; - auto type = std::make_shared(); - auto reader = make_leaf_reader(descriptor, type); - - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::FIXED_BINARY; - batch._binary_chunks = {fixed_binary_array( - {half(0x0000), half(0x8000), half(0x3E00), half(0x0001), half(0x7E00)}, 2)}; - batch._values_written = 5; - - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, nullptr, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& floats = assert_cast(*column); - ASSERT_EQ(floats.size(), 5); - EXPECT_FLOAT_EQ(floats.get_element(0), 0.0F); - EXPECT_TRUE(std::signbit(floats.get_element(1))); - EXPECT_FLOAT_EQ(floats.get_element(2), 1.5F); - EXPECT_NEAR(floats.get_element(3), 5.9604645e-8F, 1e-12F); - EXPECT_TRUE(std::isnan(floats.get_element(4))); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableValuesAreSpacedWithNullRefs) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("aa").ok()); - ASSERT_TRUE(builder.Append("cc").ok()); - ASSERT_TRUE(builder.Append("ee").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 3; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& nullable = assert_cast(*column); - const auto& strings = assert_cast(nullable.get_nested_column()); - ASSERT_EQ(nullable.size(), 5); - EXPECT_EQ(strings.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_EQ(strings.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_EQ(strings.get_data_at(4).to_string(), "ee"); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableRejectsCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("only_one").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 1; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 3, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet binary values"), - std::string::npos); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCarriesDescriptorSessionAndNullMapFields) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT64; - descriptor.time_unit = ParquetTimeUnit::NANOS; - descriptor.decimal_precision = 18; - descriptor.decimal_scale = 4; - descriptor.fixed_length = 12; - descriptor.timestamp_is_adjusted_to_utc = true; - auto type = make_nullable(std::make_shared()); - cctz::time_zone shanghai; - ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); - - CapturedDecodedView captured; - auto reader = make_spy_leaf_reader(descriptor, type, &captured, &shanghai, true); - const std::vector values = {100, 200, 300}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT64; - batch._fixed_values = reinterpret_cast(values.data()); - batch._values_written = values.size(); - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, &null_map, column).ok()); - EXPECT_EQ(captured.value_kind, DecodedValueKind::INT64); - EXPECT_EQ(captured.time_unit, DecodedTimeUnit::NANOS); - EXPECT_EQ(captured.row_count, 3); - EXPECT_EQ(captured.decimal_precision, 18); - EXPECT_EQ(captured.decimal_scale, 4); - EXPECT_EQ(captured.fixed_length, 12); - EXPECT_TRUE(captured.timestamp_is_adjusted_to_utc); - EXPECT_TRUE(captured.enable_strict_mode); - EXPECT_EQ(captured.timezone, &shanghai); - EXPECT_FALSE(captured.null_map_is_null); - EXPECT_EQ(captured.null_map, std::vector({0, 1, 0})); - - auto required_column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, nullptr, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); - - const NullMap empty_null_map; - ASSERT_TRUE(reader.append_values(batch, 3, &empty_null_map, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCapturesBinaryFixedLengthAndFloat16Override) { - ParquetTypeDescriptor binary_descriptor; - binary_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - binary_descriptor.fixed_length = 4; - auto type = std::make_shared(); - - CapturedDecodedView binary_view; - auto binary_reader = make_spy_leaf_reader(binary_descriptor, type, &binary_view); - ParquetLeafBatch binary_batch; - binary_batch._value_kind = DecodedValueKind::FIXED_BINARY; - binary_batch._binary_chunks = {fixed_binary_array({"abcd", "wxyz"}, 4)}; - binary_batch._values_written = 2; - auto binary_column = type->create_column(); - ASSERT_TRUE(binary_reader.append_values(binary_batch, 2, nullptr, binary_column).ok()); - EXPECT_EQ(binary_view.value_kind, DecodedValueKind::FIXED_BINARY); - EXPECT_EQ(binary_view.fixed_length, 4); - ASSERT_EQ(binary_view.owned_binary_values.size(), 2); - EXPECT_EQ(binary_view.owned_binary_values[0], "abcd"); - EXPECT_EQ(binary_view.owned_binary_values[1], "wxyz"); - - ParquetTypeDescriptor float16_descriptor; - float16_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - float16_descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - float16_descriptor.fixed_length = 2; - CapturedDecodedView float16_view; - auto float16_reader = make_spy_leaf_reader(float16_descriptor, - std::make_shared(), &float16_view); - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - ParquetLeafBatch float16_batch; - float16_batch._value_kind = DecodedValueKind::FIXED_BINARY; - float16_batch._binary_chunks = {fixed_binary_array({half(0x3E00), half(0x4000)}, 2)}; - float16_batch._values_written = 2; - auto float16_column = std::make_shared()->create_column(); - ASSERT_TRUE(float16_reader.append_values(float16_batch, 2, nullptr, float16_column).ok()); - EXPECT_EQ(float16_view.value_kind, DecodedValueKind::FLOAT); - ASSERT_EQ(float16_view.fixed_values.size(), sizeof(float) * 2); - const auto* floats = reinterpret_cast(float16_view.fixed_values.data()); - EXPECT_FLOAT_EQ(floats[0], 1.5F); - EXPECT_FLOAT_EQ(floats[1], 2.0F); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLevels) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 2, 2}; - const std::vector rep_levels = {0, 1, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 2, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.records_read, 2); - EXPECT_EQ(nested_batch.levels_written, 3); - EXPECT_EQ(nested_batch.value_indices, std::vector({0, 1, 2})); - const auto& nested_values = assert_cast(*nested_batch.values_column); - ASSERT_EQ(nested_values.size(), 3); - EXPECT_EQ(nested_values.get_element(0), 10); - EXPECT_EQ(nested_values.get_element(2), 30); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutValueSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 777, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 2, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLeafValues) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 1, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutPayloadSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {1, 2, 0, 2}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {777, 10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({-1, 1, -1, 2})); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsMismatchedValueLayout) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 0, 2, 0}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("inconsistent value count"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsDenseNullable) { - auto descriptor = int32_column_descriptor(1, 0); - auto reader = - make_nested_leaf_reader(descriptor, make_nullable(std::make_shared())); - const std::vector def_levels = {1}; - const std::vector rep_levels = {0}; - const std::vector values = {10}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values, true); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 1, 0, 0, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Dense nullable parquet nested reader is not supported"), - std::string::npos); -} - -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp index fac89b31c422d8..cb54cab86b27d8 100644 --- a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp +++ b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp @@ -26,95 +26,6 @@ namespace doris::format::parquet { namespace { -void expect_plan_entry(const ParquetPageCacheReadPlanEntry& entry, - const ParquetPageCacheRange& cached_range, int64_t copy_offset_in_cache, - int64_t output_offset, int64_t copy_size) { - EXPECT_EQ(entry.cached_range.offset, cached_range.offset); - EXPECT_EQ(entry.cached_range.size, cached_range.size); - EXPECT_EQ(entry.copy_offset_in_cache, copy_offset_in_cache); - EXPECT_EQ(entry.output_offset, output_offset); - EXPECT_EQ(entry.copy_size, copy_size); -} - -TEST(ParquetPageCacheRangeTest, SubsetRequestHitsSingleCachedRange) { - const std::vector cached_ranges = { - {100, 100}, - }; - - // Request [120, 150) is fully inside cached [100, 200). The reader should lookup - // the exact cached key [100, 200), then copy from cached offset 20 into output offset 0. - auto plan = detail::plan_page_cache_range_read(120, 30, cached_ranges); - - ASSERT_EQ(plan.size(), 1); - expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestHitsMultipleAdjacentCachedRanges) { - const std::vector cached_ranges = { - {180, 80}, - {100, 80}, - }; - - // Request [100, 260) is larger than either cached entry, but the two cached ranges - // exactly cover it. The copy plan stitches the two exact cache entries together. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 80}, 0, 0, 80); - expect_plan_entry(plan[1], {180, 80}, 0, 80, 80); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestCanUseOverlappingCachedRanges) { - const std::vector cached_ranges = { - {150, 110}, - {100, 100}, - }; - - // Request [100, 260) is covered by overlapping cached ranges. The first copy uses - // [100, 200); the second resumes at cursor 200 and copies the tail from [150, 260). - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 100}, 0, 0, 100); - expect_plan_entry(plan[1], {150, 110}, 50, 100, 60); -} - -TEST(ParquetPageCacheRangeTest, PartialOverlapWithoutFullCoverageMisses) { - const std::vector cached_ranges = { - {100, 80}, - {200, 60}, - }; - - // Cached ranges cover [100, 180) and [200, 260), but [180, 200) is missing. - // The caller must read the whole request from the file instead of returning - // a partially cached result. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, NonCoveringAndInvalidRangesAreIgnored) { - const std::vector cached_ranges = { - {50, 20}, {100, 0}, {100, -1}, {180, 20}, {120, 30}, - }; - - // Only [120, 150) intersects the request, but it does not cover the request start - // [100, 120), so this is still a miss. - auto plan = detail::plan_page_cache_range_read(100, 50, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, InvalidRequestMisses) { - const std::vector cached_ranges = { - {100, 100}, - }; - - EXPECT_TRUE(detail::plan_page_cache_range_read(-1, 10, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, 0, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, -1, cached_ranges).empty()); -} - TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) { const std::vector ranges = { {100, 50}, @@ -134,6 +45,19 @@ TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) EXPECT_EQ(valid_ranges[1].size, 60); } +TEST(ParquetPageCacheRangeTest, SerializedIndexesAreBoundedIndividuallyAndWhenCoalesced) { + constexpr size_t file_size = 1ULL << 30; + const auto budget = detail::MAX_SERIALIZED_PARQUET_INDEX_BYTES; + + EXPECT_TRUE(detail::is_serialized_index_range_safe(file_size, 0, budget)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, 0, budget + 1)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, -1, 1)); + + EXPECT_TRUE(detail::is_serialized_index_span_safe(100, 100 + budget)); + // Individually small adjacent indexes must not combine into one unbounded allocation. + EXPECT_FALSE(detail::is_serialized_index_span_safe(100, 101 + budget)); +} + TEST(ParquetPageCacheRangeTest, AveragePrefetchRangeSizeUsesOnlyValidRanges) { const std::vector ranges = { {0, 512}, diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index a21974bced294d..75e6545906f263 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -5,9 +5,7 @@ // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at -// // http://www.apache.org/licenses/LICENSE-2.0 -// // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -17,185 +15,47 @@ #include -#include #include #include #include #include -#include #include #include "common/consts.h" #include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" +#include "core/block/block.h" #include "core/column/column_string.h" -#include "core/column/column_struct.h" #include "core/column/column_vector.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" #include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "format_v2/parquet/selection_vector.h" #include "storage/utils.h" namespace doris::format::parquet { namespace { -ParquetColumnSchema int64_schema(std::string name = "mock") { +ParquetColumnSchema int64_schema() { ParquetColumnSchema schema; schema.local_id = 0; - schema.name = std::move(name); + schema.name = "mock"; schema.type = std::make_shared(); return schema; } -ParquetColumnSchema nested_int64_schema(std::string name, int16_t nullable_definition_level, - int16_t definition_level, int16_t repetition_level = 0, - int16_t repeated_ancestor_definition_level = 0) { - ParquetColumnSchema schema = int64_schema(std::move(name)); - schema.type = make_nullable(std::make_shared()); - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - return schema; -} - -ParquetColumnSchema nested_struct_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "struct"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})); - return schema; -} - -ParquetColumnSchema nested_list_schema(std::string name, DataTypePtr element_type, - int16_t nullable_definition_level, int16_t definition_level, - int16_t repetition_level, - int16_t repeated_ancestor_definition_level) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = std::move(name); - schema.kind = ParquetColumnSchemaKind::LIST; - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - schema.type = make_nullable(std::make_shared(std::move(element_type))); - return schema; -} - -ParquetColumnSchema nested_map_schema( - DataTypePtr value_type = make_nullable(std::make_shared())) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "map"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.repetition_level = 1; - schema.repeated_ancestor_definition_level = 2; - schema.type = make_nullable(std::make_shared( - make_nullable(std::make_shared()), std::move(value_type))); - return schema; -} - -ParquetColumnSchema bare_repeated_int64_list_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "repeated"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.definition_level = 1; - schema.repetition_level = 1; - schema.repeated_repetition_level = 1; - schema.repeated_ancestor_definition_level = 1; - schema.type = std::make_shared(std::make_shared()); - return schema; -} - -std::unique_ptr primitive_child(int local_id, std::string name, - DataTypePtr type) { - auto child = std::make_unique(); - child->local_id = local_id; - child->name = std::move(name); - child->kind = ParquetColumnSchemaKind::PRIMITIVE; - child->leaf_column_id = local_id; - child->type = std::move(type); - child->type_descriptor.physical_type = ::parquet::Type::INT32; - child->type_descriptor.doris_type = child->type; - return child; -} - -ParquetColumnSchema struct_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "s"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.children.push_back(primitive_child(0, "a", std::make_shared())); - schema.children.push_back(primitive_child(1, "b", std::make_shared())); - DataTypes types = {make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)}; - Strings names = {"a", "b"}; - schema.type = std::make_shared(types, names); - return schema; -} - -ParquetColumnSchema list_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "xs"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.children.push_back(primitive_child(0, "element", std::make_shared())); - schema.type = std::make_shared(schema.children[0]->type); - return schema; -} - -ParquetColumnSchema map_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "m"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.children.push_back(primitive_child(0, "key", std::make_shared())); - schema.children.push_back(primitive_child(1, "value", std::make_shared())); - schema.type = std::make_shared(make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)); - return schema; -} - class CursorColumnReader final : public ParquetColumnReader { public: CursorColumnReader() : ParquetColumnReader(int64_schema(), std::make_shared()) {} Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("invalid mock read arguments"); - } - auto* values = assert_cast(column.get()); + DORIS_CHECK(column); + DORIS_CHECK(rows_read != nullptr); + auto& values = assert_cast(*column); for (int64_t row = 0; row < rows; ++row) { - values->insert_value(_cursor + row); + values.insert_value(_cursor + row); } _read_lengths.push_back(rows); _cursor += rows; @@ -204,233 +64,33 @@ class CursorColumnReader final : public ParquetColumnReader { } Status skip(int64_t rows) override { + DORIS_CHECK(rows >= 0); _skip_lengths.push_back(rows); _cursor += rows; return Status::OK(); } + void flush_profile() override { ++_profile_flushes; } + bool crossed_page_since_last_batch() override { + ++_page_crossing_checks; + return _crossed_page; + } + + void set_crossed_page(bool crossed_page) { _crossed_page = crossed_page; } + int64_t cursor() const { return _cursor; } const std::vector& skip_lengths() const { return _skip_lengths; } const std::vector& read_lengths() const { return _read_lengths; } + int profile_flushes() const { return _profile_flushes; } + int page_crossing_checks() const { return _page_crossing_checks; } private: int64_t _cursor = 0; std::vector _skip_lengths; std::vector _read_lengths; -}; - -class ScriptedNestedReader final : public ParquetColumnReader { -public: - ScriptedNestedReader(ParquetColumnSchema schema, DataTypePtr type, - std::vector def_levels, std::vector rep_levels, - bool has_repeated_child = false, bool build_nulls = false) - : ParquetColumnReader(schema, std::move(type)), - _def_levels(std::move(def_levels)), - _rep_levels(std::move(rep_levels)), - _has_repeated_child(has_repeated_child), - _build_nulls(build_nulls) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - _build_lengths.push_back(length_upper_bound); - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("invalid scripted nested build arguments"); - } - for (int64_t row = 0; row < length_upper_bound; ++row) { - insert_value(column, _next_value++, _build_nulls); - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - set_nested_build_level_cursor(std::min(nested_build_level_cursor() + length_upper_bound, - static_cast(_def_levels.size()))); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return _has_repeated_child; } - - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - -private: - static void insert_value(MutableColumnPtr& column, int64_t value, bool is_null) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - if (is_null) { - nullable_column->insert_default(); - return; - } - assert_cast(nullable_column->get_nested_column()).insert_value(value); - nullable_column->get_null_map_data().push_back(0); - return; - } - assert_cast(*column).insert_value(value); - } - - std::vector _def_levels; - std::vector _rep_levels; - bool _has_repeated_child = false; - bool _build_nulls = false; - int64_t _next_value = 0; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; -}; - -class ChunkedNestedLeafReader final : public ParquetColumnReader { -public: - ChunkedNestedLeafReader() - : ParquetColumnReader(nested_int64_schema("element", 0, 1, 1, 1), - std::make_shared()) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(values_read != nullptr); - _initial_column_sizes.push_back(column->size()); - _build_lengths.push_back(length_upper_bound); - if (auto* nullable = check_and_get_column(*column); nullable != nullptr) { - auto& values = assert_cast(nullable->get_nested_column()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values.insert_value(row); - nullable->get_null_map_data().push_back(0); - } - } else { - auto* values = assert_cast(column.get()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values->insert_value(row); - } - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return true; } - - const std::vector& load_lengths() const { return _load_lengths; } - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - const std::vector& initial_column_sizes() const { return _initial_column_sizes; } - -private: - std::vector _def_levels; - std::vector _rep_levels; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; - std::vector _initial_column_sizes; -}; - -} // namespace - -struct ScalarColumnReaderTestAccess { - static void set_nested_batch(ScalarColumnReader* reader, - std::unique_ptr batch) { - reader->_nested_batch = std::move(batch); - } - - static int64_t page_filtered_rows_to_skip(const ScalarColumnReader& reader, int64_t rows) { - return reader.page_filtered_rows_to_skip(rows); - } - - static void set_row_group_rows_read(ScalarColumnReader* reader, int64_t rows) { - reader->_row_group_rows_read = rows; - } -}; - -namespace { - -std::unique_ptr make_scripted_scalar_reader( - ParquetColumnSchema schema, std::unique_ptr batch) { - auto reader = std::make_unique(schema, nullptr); - ScalarColumnReaderTestAccess::set_nested_batch(reader.get(), std::move(batch)); - return reader; -} - -std::unique_ptr scalar_batch(std::vector def_levels, - std::vector rep_levels, - std::vector value_indices, - std::vector values) { - auto batch = std::make_unique(); - batch->levels_written = static_cast(def_levels.size()); - batch->def_levels = std::move(def_levels); - batch->rep_levels = std::move(rep_levels); - batch->value_indices = std::move(value_indices); - auto column = ColumnInt64::create(); - for (const auto value : values) { - column->insert_value(value); - } - batch->values_column = std::move(column); - return batch; -} - -class DefaultOnlyReader final : public ParquetColumnReader { -public: - DefaultOnlyReader() - : ParquetColumnReader(int64_schema("default_only"), std::make_shared()) { - } - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } + int _profile_flushes = 0; + bool _crossed_page = false; + int _page_crossing_checks = 0; }; GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { @@ -466,6 +126,21 @@ TEST(SelectionVectorTest, ExternalBufferSelectionToRanges) { EXPECT_TRUE(selection.verify(std::size(indices), 8).ok()); } +TEST(SelectionVectorTest, OutputRangesReuseCapacity) { + SelectionVector::Index indices[] = {1, 2, 5}; + SelectionVector selection(indices, std::size(indices)); + std::vector ranges; + ranges.reserve(8); + const auto retained_capacity = ranges.capacity(); + + selection_to_ranges(selection, std::size(indices), &ranges); + ASSERT_EQ(ranges.size(), 2); + EXPECT_EQ(ranges.capacity(), retained_capacity); + selection_to_ranges(selection, 1, &ranges); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges.capacity(), retained_capacity); +} + TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { SelectionVector selection(2); EXPECT_FALSE(selection.verify(3, 3).ok()); @@ -480,6 +155,28 @@ TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { EXPECT_FALSE(selection.verify(2, 3).ok()); } +TEST(SelectionVectorTest, MaterializedFilterIsReusedUntilSelectionChanges) { + SelectionVector selection(4); + selection.set_index(0, 1); + selection.set_index(1, 3); + const uint8_t* first_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &first_filter).ok()); + ASSERT_NE(first_filter, nullptr); + EXPECT_EQ(std::vector(first_filter, first_filter + 4), + std::vector({0, 1, 0, 1})); + + const uint8_t* reused_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &reused_filter).ok()); + EXPECT_EQ(reused_filter, first_filter); + + selection.set_index(1, 2); + const uint8_t* updated_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &updated_filter).ok()); + EXPECT_EQ(updated_filter, first_filter); + EXPECT_EQ(std::vector(updated_filter, updated_filter + 4), + std::vector({0, 1, 1, 0})); +} + TEST(ParquetColumnReaderControlTest, BaseSelectUsesSkipReadRanges) { CursorColumnReader reader; SelectionVector selection(3); @@ -505,571 +202,67 @@ TEST(ParquetColumnReaderControlTest, BaseSelectZeroRowsConsumesBatch) { SelectionVector selection; auto column = std::make_shared()->create_column(); ASSERT_TRUE(reader.select(selection, 0, 4, column).ok()); - EXPECT_EQ(column->size(), 0); + EXPECT_TRUE(column->empty()); EXPECT_EQ(reader.cursor(), 4); EXPECT_TRUE(reader.read_lengths().empty()); EXPECT_EQ(reader.skip_lengths(), std::vector({4})); } -TEST(ParquetColumnReaderControlTest, BaseNestedDefaultsAndSkipNested) { - DefaultOnlyReader base_reader; - EXPECT_FALSE(base_reader.skip(1).ok()); - EXPECT_FALSE(base_reader.load_nested_batch(1).ok()); - - auto column = std::make_shared()->create_column(); - int64_t values_read = 0; - EXPECT_FALSE(base_reader.build_nested_column(1, column, &values_read).ok()); - - int64_t values_consumed = 0; - EXPECT_FALSE(base_reader.consume_nested_column(1, &values_consumed).ok()); -} - -TEST(ParquetColumnReaderControlTest, NestedSkipConsumesBoundedBatchesWithoutMaterializing) { - auto element_reader = std::make_unique(); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); - - ASSERT_TRUE(reader.skip(8193).ok()); - EXPECT_TRUE(element_reader_ptr->load_lengths().empty()); - EXPECT_EQ(element_reader_ptr->level_load_lengths(), std::vector({4096, 4096, 1})); - EXPECT_EQ(element_reader_ptr->consume_lengths(), std::vector({4096, 4096, 1})); - EXPECT_TRUE(element_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(element_reader_ptr->initial_column_sizes().empty()); -} - -TEST(ParquetColumnReaderControlTest, MapSkipConsumesBothStreamsWithoutMaterializing) { - const std::vector def_levels {3, 3, 1, 3}; - const std::vector rep_levels {0, 1, 0, 0}; - auto key_reader = std::make_unique( - nested_int64_schema("key", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* key_reader_ptr = key_reader.get(); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); +TEST(ParquetColumnReaderControlTest, SchedulerFlushesReaderProfilesAtBatchBoundary) { + ParquetScanScheduler scheduler; + auto reader = std::make_unique(); + auto* reader_ptr = reader.get(); + scheduler._current_predicate_columns.emplace(0, std::move(reader)); - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(key_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(key_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_TRUE(key_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(value_reader_ptr->build_lengths().empty()); + scheduler.flush_current_reader_profiles(); + EXPECT_EQ(reader_ptr->profile_flushes(), 1); } -TEST(ParquetColumnReaderControlTest, StructSkipConsumesNullSeparatedChildSpans) { - const std::vector def_levels {2, 0, 2}; - const std::vector rep_levels {0, 0, 0}; - auto shape_reader = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* shape_reader_ptr = shape_reader.get(); - auto child_reader = std::make_unique( - nested_int64_schema("child", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* child_reader_ptr = child_reader.get(); - std::vector> children; - children.push_back(std::move(shape_reader)); - children.push_back(std::move(child_reader)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {-1, 0}); - - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(shape_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(child_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(shape_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_EQ(child_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_TRUE(shape_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(child_reader_ptr->build_lengths().empty()); -} +TEST(ParquetColumnReaderControlTest, SchedulerOrsPageCrossingOncePerBatch) { + ParquetScanScheduler scheduler; + auto predicate_reader = std::make_unique(); + auto* predicate_ptr = predicate_reader.get(); + predicate_ptr->set_crossed_page(true); + scheduler._current_predicate_columns.emplace(0, std::move(predicate_reader)); -TEST(ParquetColumnReaderControlTest, NestedListSkipConsumesRecursivelyWithoutMaterializing) { - auto leaf_reader = std::make_unique( - nested_int64_schema("leaf", 0, 1, 1, 1), std::make_shared(), - std::vector {1, 1}, std::vector {0, 0}); - auto* leaf_reader_ptr = leaf_reader.get(); - const auto inner_type = std::make_shared(std::make_shared()); - auto inner_reader = std::make_unique(bare_repeated_int64_list_schema(), - inner_type, std::move(leaf_reader)); - const auto outer_type = std::make_shared(inner_type); - ListColumnReader reader(bare_repeated_int64_list_schema(), outer_type, std::move(inner_reader)); + auto lazy_reader = std::make_unique(); + auto* lazy_ptr = lazy_reader.get(); + lazy_ptr->set_crossed_page(true); + scheduler._current_non_predicate_columns.emplace(1, std::move(lazy_reader)); - ASSERT_TRUE(reader.skip(2).ok()); - EXPECT_EQ(leaf_reader_ptr->level_load_lengths(), std::vector({2})); - EXPECT_EQ(leaf_reader_ptr->consume_lengths(), std::vector({2})); - EXPECT_TRUE(leaf_reader_ptr->build_lengths().empty()); + // Both readers are sampled even after the OR becomes true so their next batch starts cleanly. + EXPECT_TRUE(scheduler.finish_current_reader_batch_profiles()); + EXPECT_EQ(predicate_ptr->page_crossing_checks(), 1); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 1); } -TEST(ParquetColumnReaderControlTest, NestedMaterializerHelpersAppendOffsetsAndParentNulls) { - ColumnArray::Offsets64 offsets; - append_offsets(offsets, {3, 0, 2}); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 3); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 5); - append_offsets(offsets, {1, 4}); - ASSERT_EQ(offsets.size(), 5); - EXPECT_EQ(offsets[3], 6); - EXPECT_EQ(offsets[4], 10); - - const NullMap parent_nulls = {0, 1, 0}; - append_parent_nulls(nullptr, parent_nulls); - NullMap dst = {1}; - append_parent_nulls(&dst, parent_nulls); - EXPECT_EQ(dst, NullMap({1, 0, 1, 0})); -} - -TEST(ParquetColumnReaderControlTest, PageFilteredRowsToSkipUsesOnlyFullSkippedRanges) { - ParquetPageSkipPlan page_skip_plan; - page_skip_plan.skipped_ranges = {RowRange {0, 3}, RowRange {5, 2}, RowRange {10, 4}}; - - auto schema = nested_int64_schema("page_filtered", 0, 0); - ScalarColumnReader reader(schema, nullptr, &page_skip_plan); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 3), 3); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 3); - - ScalarColumnReaderTestAccess::set_row_group_rows_read(&reader, 5); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 2), 2); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 2); -} - -TEST(ParquetColumnReaderControlTest, StructSkipsNullParentForRepeatedChildAndBatchesPresentRows) { - auto repeated_child = std::make_unique( - nested_int64_schema("repeated_shape", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2, 2, 2}, - std::vector {0, 0, 0, 0}, true); - auto* repeated_child_ptr = repeated_child.get(); - auto scalar_child = make_scripted_scalar_reader( - nested_int64_schema("scalar_child", 1, 2), - scalar_batch({2, 0, 2, 2}, {0, 0, 0, 0}, {0, -1, 1, 2}, {10, 20, 30})); - auto* scalar_child_ptr = scalar_child.get(); - - std::vector> children; - children.push_back(std::move(repeated_child)); - children.push_back(std::move(scalar_child)); - StructColumnReader reader(nested_struct_schema(), - make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})), - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_EQ(repeated_child_ptr->build_lengths(), std::vector({1, 2})); - EXPECT_EQ(scalar_child_ptr->nested_build_level_cursor(), 4); -} - -TEST(ParquetColumnReaderControlTest, StructFallsBackToFirstChildWhenAllChildrenAreRepeated) { - auto first_child = std::make_unique( - nested_int64_schema("first", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 0}, std::vector {0, 0}, true); - auto second_child = std::make_unique( - nested_int64_schema("second", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2}, - std::vector {0, 0}, true); - - std::vector> children; - children.push_back(std::move(first_child)); - children.push_back(std::move(second_child)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(rows_read, 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesComplexChildShapeOnly) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 2, 0, 0, 2, 2}, std::vector {0, 0, 0, 0, 0, 0}); - - ParquetColumnSchema map_schema = nested_map_schema(); - map_schema.nullable_definition_level = 2; - map_schema.definition_level = 3; - map_schema.repeated_ancestor_definition_level = 0; - auto key_reader = std::make_unique( - nested_int64_schema("key", 3, 3, 1, 0), - make_nullable(std::make_shared()), - std::vector {3, 3, 0, 0, 3, 3}, std::vector {0, 0, 0, 0, 0, 0}); - auto value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 4, 4, 1, 0), - scalar_batch({4, 4, 0, 0, 4, 4}, {0, 0, 0, 0, 0, 0}, - {0, 1, -1, -1, 2, 3}, {10, 20, 30, 40})); - auto map_reader = std::make_unique( - map_schema, map_schema.type, std::move(key_reader), std::move(value_reader)); - - std::vector> children; - children.push_back(std::move(shape_child)); - children.push_back(std::move(map_reader)); - auto struct_type = make_nullable(std::make_shared(DataTypes {map_schema.type}, - Strings {"partitionValues"})); - StructColumnReader reader(nested_struct_schema(), struct_type, std::move(children), {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(6, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 6); - - const auto& nullable_struct = assert_cast(*column); - ASSERT_EQ(nullable_struct.size(), 6); - EXPECT_FALSE(nullable_struct.is_null_at(0)); - EXPECT_FALSE(nullable_struct.is_null_at(1)); - EXPECT_TRUE(nullable_struct.is_null_at(2)); - EXPECT_TRUE(nullable_struct.is_null_at(3)); - EXPECT_FALSE(nullable_struct.is_null_at(4)); - EXPECT_FALSE(nullable_struct.is_null_at(5)); - - const auto& struct_column = - assert_cast(nullable_struct.get_nested_column()); - const auto& map_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_nullable.size(), 6); - EXPECT_FALSE(map_nullable.is_null_at(0)); - EXPECT_FALSE(map_nullable.is_null_at(1)); - EXPECT_TRUE(map_nullable.is_null_at(2)); - EXPECT_TRUE(map_nullable.is_null_at(3)); - EXPECT_FALSE(map_nullable.is_null_at(4)); - EXPECT_FALSE(map_nullable.is_null_at(5)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 6); - EXPECT_EQ(map_column.get_offsets()[0], 1); - EXPECT_EQ(map_column.get_offsets()[1], 2); - EXPECT_EQ(map_column.get_offsets()[2], 2); - EXPECT_EQ(map_column.get_offsets()[3], 2); - EXPECT_EQ(map_column.get_offsets()[4], 3); - EXPECT_EQ(map_column.get_offsets()[5], 4); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesNestedStructDescendants) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 0, 2}, std::vector {0, 0, 0}); - - auto id_batch = scalar_batch({4, 3, 4}, {0, 0, 0}, {0, -1, 1}, {10, 20}); - id_batch->value_slot_definition_level = 3; - auto id_reader = - make_scripted_scalar_reader(nested_int64_schema("id", 3, 4), std::move(id_batch)); - - ParquetColumnSchema inner_schema; - inner_schema.local_id = 0; - inner_schema.name = "stats_parsed"; - inner_schema.kind = ParquetColumnSchemaKind::STRUCT; - inner_schema.nullable_definition_level = 2; - inner_schema.definition_level = 3; - inner_schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared())}, Strings {"id"})); - - std::vector> inner_children; - inner_children.push_back(std::move(id_reader)); - auto inner_reader = std::make_unique( - inner_schema, inner_schema.type, std::move(inner_children), std::vector {0}); - - std::vector> outer_children; - outer_children.push_back(std::move(shape_child)); - outer_children.push_back(std::move(inner_reader)); - auto outer_type = make_nullable(std::make_shared(DataTypes {inner_schema.type}, - Strings {"stats_parsed"})); - StructColumnReader reader(nested_struct_schema(), outer_type, std::move(outer_children), - {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); +TEST(ParquetColumnReaderControlTest, PendingOutputDrainsBeforePageCrossingSample) { + ParquetScanScheduler scheduler; + scheduler._batch_size = 1; + auto lazy_reader = std::make_unique(); + auto* lazy_ptr = lazy_reader.get(); + scheduler._current_non_predicate_columns.emplace(format::LocalColumnId(0), + std::move(lazy_reader)); + scheduler._pending_predicate_batch_rows = 2; + scheduler._pending_predicate_selection = {0, 1}; - const auto& outer_nullable = assert_cast(*column); - ASSERT_EQ(outer_nullable.size(), 3); - EXPECT_FALSE(outer_nullable.is_null_at(0)); - EXPECT_TRUE(outer_nullable.is_null_at(1)); - EXPECT_FALSE(outer_nullable.is_null_at(2)); + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + Block block; + auto type = std::make_shared(); + block.insert({type->create_column(), type, "mock"}); + size_t rows = 0; - const auto& outer_struct = assert_cast(outer_nullable.get_nested_column()); - const auto& inner_nullable = assert_cast(outer_struct.get_column(0)); - ASSERT_EQ(inner_nullable.size(), 3); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_TRUE(inner_nullable.is_null_at(1)); - EXPECT_FALSE(inner_nullable.is_null_at(2)); + ASSERT_TRUE(scheduler.materialize_pending_predicate_batch(request, &block, &rows).ok()); + EXPECT_EQ(rows, 1); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 0); + lazy_ptr->set_crossed_page(true); - const auto& inner_struct = assert_cast(inner_nullable.get_nested_column()); - const auto& id_nullable = assert_cast(inner_struct.get_column(0)); - const auto& id_values = assert_cast(id_nullable.get_nested_column()); - EXPECT_EQ(id_values.get_element(0), 10); - EXPECT_EQ(id_values.get_element(2), 20); -} - -TEST(ParquetColumnReaderControlTest, ListKeepsEmptyBareRepeatedPrimitiveRows) { - auto element_reader = std::make_unique( - nested_int64_schema("element", 0, 1, 1, 1), std::make_shared(), - std::vector {0, 1, 1, 0}, std::vector {0, 0, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); - - const auto& array_column = assert_cast(*column); - ASSERT_EQ(array_column.get_offsets().size(), 3); - EXPECT_EQ(array_column.get_offsets()[0], 0); - EXPECT_EQ(array_column.get_offsets()[1], 2); - EXPECT_EQ(array_column.get_offsets()[2], 2); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({2})); -} - -TEST(ParquetColumnReaderControlTest, NestedListSkipsAncestorEmptyRowsButKeepsNullElements) { - auto element_reader = - std::make_unique(nested_int64_schema("element", 5, 5, 2, 4), - make_nullable(std::make_shared()), - std::vector {1, 5, 5, 5, 2, 5, 2, 0}, - std::vector {0, 0, 2, 1, 0, 1, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - - const auto inner_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()))); - auto inner_reader = std::make_unique( - nested_list_schema("inner", make_nullable(std::make_shared()), 3, 4, 2, - 2), - inner_type, std::move(element_reader)); - auto outer_type = make_nullable(std::make_shared(inner_type)); - ListColumnReader reader(nested_list_schema("outer", inner_type, 1, 2, 1, 2), outer_type, - std::move(inner_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 5); - EXPECT_EQ(outer_offsets[3], 5); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 5); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_TRUE(inner_nullable.is_null_at(4)); - - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 5); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 3); - EXPECT_EQ(inner_offsets[2], 3); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 4); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({4})); -} - -TEST(ParquetColumnReaderControlTest, MapKeepsEmptyMapRows) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 1); - - const auto& nullable_map = assert_cast(*column); - EXPECT_FALSE(nullable_map.is_null_at(0)); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(value_reader_ptr->build_lengths(), std::vector({0})); -} - -TEST(ParquetColumnReaderControlTest, ListMapSkipsAncestorEmptyRowsBeforeScalarValues) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 4, 4, 2, 4), - make_nullable(std::make_shared()), std::vector {1, 4}, - std::vector {0, 0}); - auto value_reader = make_scripted_scalar_reader(nested_int64_schema("value", 5, 5, 2, 4), - scalar_batch({1, 5}, {0, 0}, {-1, 0}, {100})); - - const auto map_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()), - make_nullable(std::make_shared()))); - auto map_reader = std::make_unique( - nested_map_schema(make_nullable(std::make_shared())), map_type, - std::move(key_reader), std::move(value_reader)); - auto outer_type = make_nullable(std::make_shared(map_type)); - ListColumnReader reader(nested_list_schema("outer", map_type, 1, 2, 1, 2), outer_type, - std::move(map_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 2); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 2); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 1); - - const auto& map_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(map_nullable.size(), 1); - EXPECT_FALSE(map_nullable.is_null_at(0)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 1); - - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 1); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_EQ(value_data.get_element(0), 100); -} - -TEST(ParquetColumnReaderControlTest, MapRejectsNullKeysAndMisalignedScalarValueRepLevels) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}, false, true); - auto value_reader = std::make_unique( - nested_int64_schema("value", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}); - MapColumnReader null_key_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(value_reader)); - auto column = null_key_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = null_key_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null key"), std::string::npos); - - auto aligned_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(aligned_key_reader), - std::move(misaligned_value_reader)); - column = misaligned_reader.type()->create_column(); - status = misaligned_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} - -TEST(ParquetColumnReaderControlTest, MapConsumePreservesKeyAndValueCorruptionChecks) { - auto null_key_reader = - make_scripted_scalar_reader(nested_int64_schema("key", 2, 3, 1, 2), - scalar_batch({2}, {0}, {-1}, std::vector {})); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {2}, - std::vector {0}); - MapColumnReader null_key_reader_map(nested_map_schema(), nested_map_schema().type, - std::move(null_key_reader), std::move(value_reader)); - int64_t values_consumed = 0; - auto status = null_key_reader_map.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null"), std::string::npos); - - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(misaligned_value_reader)); - status = misaligned_reader.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} - -TEST(ParquetColumnReaderControlTest, MapBuildsScalarAndComplexValuePaths) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto scalar_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 1}, {0, 1}, {100, 200})); - MapColumnReader scalar_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(scalar_value_reader)); - auto column = scalar_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = scalar_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_map = assert_cast(*column); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 2); - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 2); - EXPECT_EQ(value_data.get_element(0), 100); - EXPECT_EQ(value_data.get_element(1), 200); - - auto complex_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto complex_value_reader = std::make_unique( - nested_int64_schema("complex_value", 2, 3, 1), - make_nullable(std::make_shared()), std::vector {3, 3}, - std::vector {0, 1}); - auto* complex_value_reader_ptr = complex_value_reader.get(); - MapColumnReader complex_reader(nested_map_schema(), nested_map_schema().type, - std::move(complex_key_reader), std::move(complex_value_reader)); - column = complex_reader.type()->create_column(); - status = complex_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(complex_value_reader_ptr->build_lengths(), std::vector({2})); + block.get_by_position(0).column = type->create_column(); + ASSERT_TRUE(scheduler.materialize_pending_predicate_batch(request, &block, &rows).ok()); + EXPECT_EQ(rows, 1); + EXPECT_TRUE(scheduler._pending_predicate_selection.empty()); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 1); } TEST(ParquetVirtualColumnReaderTest, RowPositionReadSkipAndInvalidArgs) { @@ -1138,65 +331,4 @@ TEST(ParquetVirtualColumnReaderTest, GlobalRowIdReadSkipSelectAndInvalidArgs) { EXPECT_FALSE(reader.read(1, column, nullptr).ok()); } -TEST(ParquetColumnReaderFactoryTest, RejectsInvalidLeafIdBeforeCreatingRecordReader) { - ParquetColumnSchema schema = int64_schema("bad_leaf"); - schema.kind = ParquetColumnSchemaKind::PRIMITIVE; - schema.leaf_column_id = 3; - schema.type_descriptor.physical_type = ::parquet::Type::INT64; - schema.type_descriptor.doris_type = schema.type; - - ParquetColumnReaderFactory factory(nullptr, 1); - std::unique_ptr reader; - const auto status = factory.create(schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsStructInvalidAndEmptyProjection) { - auto schema = struct_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto empty_projection = format::LocalColumnIndex::partial_local(0); - status = factory.create(schema, &empty_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsListProjectionWithoutElement) { - auto schema = list_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto projection = format::LocalColumnIndex::partial_local(0); - const auto status = factory.create(schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsMapInvalidAndKeyOnlyProjection) { - auto schema = map_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(1)); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto key_only_projection = format::LocalColumnIndex::partial_local(0); - key_only_projection.children.push_back(format::LocalColumnIndex::local(0)); - status = factory.create(schema, &key_only_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no value"), std::string::npos); -} - } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 796123832c6f16..adb3b4b383c9c8 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -35,10 +36,15 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_decimal.h" +#include "core/column/column_map.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -58,6 +64,7 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -66,6 +73,7 @@ #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -142,6 +150,86 @@ class Int32GreaterThanExpr final : public VExpr { const std::string _expr_name = "Int32GreaterThanExpr"; }; +class Int32DictionaryEqualsExpr final : public VExpr { +public: + Int32DictionaryEqualsExpr(int column_id, int32_t value) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& input = nullable_nested_column(*block, _column_id); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = input.get_element(input_row) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + const auto* dictionary = ctx.slot(_column_id); + if (dictionary == nullptr) { + return ZoneMapFilterResult::kUnsupported; + } + const auto expected = Field::create_field(_value); + return std::ranges::any_of(dictionary->values, + [&](const Field& value) { return value == expected; }) + ? ZoneMapFilterResult::kMayMatch + : ZoneMapFilterResult::kNoMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + const int32_t _value; + const std::string _expr_name = "Int32DictionaryEqualsExpr"; +}; + +class DictionaryAcceptAllExpr final : public VExpr { +public: + explicit DictionaryAcceptAllExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) {} + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter( + const DictionaryEvalContext& ctx) const override { + return ctx.slot(_column_id) == nullptr ? ZoneMapFilterResult::kUnsupported + : ZoneMapFilterResult::kMayMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + const int _column_id; + const std::string _expr_name = "DictionaryAcceptAllExpr"; +}; + class Int32SumGreaterThanExpr final : public VExpr { public: Int32SumGreaterThanExpr(int left_column_id, int right_column_id, int32_t value) @@ -376,6 +464,21 @@ VExprContextSPtr create_int32_greater_than_conjunct(int column_id, int32_t value return ctx; } +VExprContextSPtr create_int32_dictionary_equals_conjunct(int column_id, int32_t value) { + auto ctx = VExprContext::create_shared( + std::make_shared(column_id, value)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + +VExprContextSPtr create_dictionary_accept_all_conjunct(int column_id) { + auto ctx = VExprContext::create_shared(std::make_shared(column_id)); + ctx->_prepared = true; + ctx->_opened = true; + return ctx; +} + VExprContextSPtr create_int32_sum_greater_than_conjunct(int left_column_id, int right_column_id, int32_t value) { auto ctx = VExprContext::create_shared( @@ -458,6 +561,39 @@ std::shared_ptr build_int32_array(const std::vector& valu return finish_array(&builder); } +std::shared_ptr build_nullable_int32_array( + const std::vector>& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(value.has_value() ? builder.Append(*value).ok() : builder.AppendNull().ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_int64_array(const std::vector& values) { + arrow::Int64Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_float_array(const std::vector& values) { + arrow::FloatBuilder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_double_array(const std::vector& values) { + arrow::DoubleBuilder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -466,6 +602,14 @@ std::shared_ptr build_string_array(const std::vector& return finish_array(&builder); } +std::shared_ptr build_binary_array(const std::vector& values) { + arrow::BinaryBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_timestamp_array(const std::shared_ptr& type, const std::vector& values) { arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); @@ -475,6 +619,27 @@ std::shared_ptr build_timestamp_array(const std::shared_ptr build_decimal_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); + for (const auto value : values) { + EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_fixed_binary_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); + const int32_t byte_width = + std::static_pointer_cast(type)->byte_width(); + for (const auto& value : values) { + EXPECT_EQ(value.size(), byte_width); + EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_struct_array(const std::vector& ids, const std::vector& names) { auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false), @@ -517,6 +682,41 @@ void write_parquet_file(const std::string& file_path, int64_t row_group_size = R row_group_size, builder.build())); } +void write_unannotated_binary_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("raw_bytes", arrow::binary(), false)}); + auto table = arrow::Table::Make(schema, {build_binary_array({"否", "是", "测试"})}); + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 3, + builder.build())); +} + +void write_decimal_and_fixed_binary_parquet_file(const std::string& file_path) { + auto decimal_type = arrow::decimal128(38, 6); + auto fixed_type = arrow::fixed_size_binary(4); + auto schema = arrow::schema({arrow::field("decimal_value", decimal_type, false), + arrow::field("fixed_value", fixed_type, false)}); + auto table = arrow::Table::Make( + schema, + {build_decimal_array(decimal_type, {1234567, -1, 0, -987654321, 42}), + build_fixed_binary_array(fixed_type, {"ABCD", std::string("\0x\0y", 4), "wxyz", "1234", + std::string("\xff\x00\x7f\x80", 4)})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.disable_dictionary(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + std::shared_ptr build_nullable_int_string_map_array() { auto key_builder = std::make_shared(); auto value_builder = std::make_shared(); @@ -686,6 +886,27 @@ void write_nullable_string_struct_parquet_file(const std::string& file_path) { ROW_COUNT, builder.build())); } +void write_nullable_complex_parquet_file(const std::string& file_path) { + auto map_array = build_nullable_int_string_map_array(); + auto list_array = build_nullable_string_list_array(); + auto struct_array = build_nullable_string_struct_array(); + auto table = arrow::Table::Make(arrow::schema({arrow::field("m", map_array->type(), true), + arrow::field("a", list_array->type(), true), + arrow::field("s", struct_array->type(), true)}), + {map_array, list_array, struct_array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_nullable_struct_with_list_parquet_file(const std::string& file_path) { auto scalar_first = build_nullable_struct_with_list_array(false); auto list_first = build_nullable_struct_with_list_array(true); @@ -706,6 +927,34 @@ void write_nullable_struct_with_list_parquet_file(const std::string& file_path) ROW_COUNT, builder.build())); } +void write_nested_complex_under_struct_parquet_file(const std::string& file_path) { + auto nested_struct = build_nullable_string_struct_array(); + auto nested_array = build_nullable_string_list_array(); + auto nested_map = build_nullable_int_string_map_array(); + auto marker = build_int32_array({10, 20, 30, 40, 50}); + auto struct_field = arrow::field("nested_struct", nested_struct->type(), true); + auto array_field = arrow::field("nested_array", nested_array->type(), true); + auto map_field = arrow::field("nested_map", nested_map->type(), true); + auto marker_field = arrow::field("marker", arrow::int32(), false); + auto outer_result = + arrow::StructArray::Make({nested_struct, nested_array, nested_map, marker}, + {struct_field, array_field, map_field, marker_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + auto outer = *outer_result; + auto table = arrow::Table::Make(arrow::schema({arrow::field("outer", outer->type(), false)}), + {outer}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_int96_timestamp_parquet_file(const std::string& file_path) { auto field = arrow::field("ts_tz", arrow::timestamp(arrow::TimeUnit::MICRO), true); auto array = @@ -792,14 +1041,21 @@ void write_struct_filter_parquet_file(const std::string& file_path) { builder.build())); } -void write_dictionary_filter_parquet_file(const std::string& file_path) { +void write_dictionary_filter_parquet_file( + const std::string& file_path, + ::parquet::Compression::type compression = ::parquet::Compression::UNCOMPRESSED) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("value", arrow::utf8(), false), }); - auto table = - arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), - build_string_array({"aa", "az", "lm", "lz", "za", "zz"})}); + const std::vector values = + compression == ::parquet::Compression::UNCOMPRESSED + ? std::vector {"aa", "az", "lm", "lz", "za", "zz"} + : std::vector {std::string(4096, 'a'), std::string(4096, 'b'), + std::string(4096, 'c'), std::string(4096, 'd'), + std::string(4096, 'e'), std::string(4096, 'f')}; + auto table = arrow::Table::Make( + schema, {build_int32_array({1, 2, 3, 4, 5, 6}), build_string_array(values)}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -808,7 +1064,7 @@ void write_dictionary_filter_parquet_file(const std::string& file_path) { ::parquet::WriterProperties::Builder builder; builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.compression(compression); builder.enable_dictionary("value"); builder.disable_dictionary("id"); builder.disable_statistics(); @@ -840,16 +1096,14 @@ void write_single_row_group_dictionary_filter_parquet_file(const std::string& fi builder.build())); } -void write_dictionary_filter_with_trailing_column_parquet_file(const std::string& file_path) { +void write_fixed_width_dictionary_filter_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), - arrow::field("value", arrow::utf8(), false), - arrow::field("payload", arrow::int32(), false), + arrow::field("value", arrow::int32(), true), }); - auto table = - arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), - build_string_array({"aa", "az", "lm", "lz", "za", "zz"}), - build_int32_array({10, 20, 30, 40, 50, 60})}); + auto table = arrow::Table::Make( + schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_nullable_int32_array({10, 20, std::nullopt, 20, 40, 20})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -861,46 +1115,62 @@ void write_dictionary_filter_with_trailing_column_parquet_file(const std::string builder.compression(::parquet::Compression::UNCOMPRESSED); builder.disable_dictionary("id"); builder.enable_dictionary("value"); - builder.disable_dictionary("payload"); builder.disable_statistics(); PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, builder.build())); } -void write_nested_dictionary_filter_parquet_file(const std::string& file_path) { - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); +void write_all_fixed_width_dictionary_filter_parquet_file(const std::string& file_path) { + auto fixed_type = arrow::fixed_size_binary(4); + auto timestamp_type = arrow::timestamp(arrow::TimeUnit::MICRO); auto schema = arrow::schema({ - arrow::field("s", struct_type, false), + arrow::field("id", arrow::int32(), false), + arrow::field("int64_value", arrow::int64(), false), + arrow::field("float_value", arrow::float32(), false), + arrow::field("double_value", arrow::float64(), false), + arrow::field("fixed_value", fixed_type, false), + arrow::field("int96_value", timestamp_type, false), }); auto table = arrow::Table::Make( - schema, {build_struct_array({1, 2, 3, 4, 5, 6}, {"aa", "az", "lm", "lz", "za", "zz"})}); + schema, + {build_int32_array({1, 2, 3, 4, 5, 6}), build_int64_array({10, 20, 10, 30, 20, 10}), + build_float_array({1.5F, 2.5F, 1.5F, 3.5F, 2.5F, 1.5F}), + build_double_array({10.25, 20.25, 10.25, 30.25, 20.25, 10.25}), + build_fixed_binary_array(fixed_type, {"AAAA", "BBBB", "AAAA", "CCCC", "BBBB", "AAAA"}), + build_timestamp_array(timestamp_type, {1000, 2000, 1000, 3000, 2000, 1000})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); std::shared_ptr out = *file_result; - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.enable_dictionary("s.name"); - builder.disable_dictionary("s.identifier.field_id"); - builder.disable_statistics(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, - builder.build())); + ::parquet::WriterProperties::Builder writer_builder; + writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); + writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + writer_builder.compression(::parquet::Compression::UNCOMPRESSED); + writer_builder.disable_dictionary("id"); + writer_builder.enable_dictionary("int64_value"); + writer_builder.enable_dictionary("float_value"); + writer_builder.enable_dictionary("double_value"); + writer_builder.enable_dictionary("fixed_value"); + writer_builder.enable_dictionary("int96_value"); + writer_builder.disable_statistics(); + ::parquet::ArrowWriterProperties::Builder arrow_builder; + arrow_builder.enable_force_write_int96_timestamps(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, + writer_builder.build(), + arrow_builder.build())); } -void write_dictionary_edge_parquet_file(const std::string& file_path) { +void write_dictionary_filter_with_trailing_column_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("value", arrow::utf8(), false), + arrow::field("payload", arrow::int32(), false), }); - auto table = arrow::Table::Make( - schema, - {build_int32_array({1, 2, 3, 4, 5, 6, 7, 8}), - build_string_array({"", "same", "other", "long-value", "", "tail", "same", "last"})}); + auto table = + arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), + build_string_array({"aa", "az", "lm", "lz", "za", "zz"}), + build_int32_array({10, 20, 30, 40, 50, 60})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -910,52 +1180,23 @@ void write_dictionary_edge_parquet_file(const std::string& file_path) { builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.enable_dictionary("value"); builder.disable_dictionary("id"); + builder.enable_dictionary("value"); + builder.disable_dictionary("payload"); builder.disable_statistics(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 6, builder.build())); } -void write_nested_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); - std::vector names; - names.reserve(ids.size()); - for (const auto id : ids) { - names.push_back("name-" + std::to_string(id)); - } - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); - auto schema = arrow::schema({ - arrow::field("s", struct_type, false), - }); - auto table = arrow::Table::Make(schema, {build_struct_array(ids, names)}); - - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); -} - -void write_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); +void write_dictionary_edge_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), + arrow::field("value", arrow::utf8(), false), }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + auto table = arrow::Table::Make( + schema, + {build_int32_array({1, 2, 3, 4, 5, 6, 7, 8}), + build_string_array({"", "same", "other", "long-value", "", "tail", "same", "last"})}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -965,12 +1206,11 @@ void write_page_index_filter_parquet_file(const std::string& file_path) { builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); + builder.enable_dictionary("value"); + builder.disable_dictionary("id"); + builder.disable_statistics(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); } void write_page_index_filter_pair_parquet_file(const std::string& file_path) { @@ -1136,7 +1376,8 @@ class NewParquetReaderTest : public testing::Test { RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, - bool is_immutable = false) const { + bool is_immutable = false, bool enable_mapping_varbinary = false, + std::string fs_name = {}, int64_t mtime = 0) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); @@ -1145,9 +1386,11 @@ class NewParquetReaderTest : public testing::Test { file_description->range_start_offset = range_start_offset; file_description->range_size = range_size; file_description->is_immutable = is_immutable; + file_description->fs_name = std::move(fs_name); + file_description->mtime = mtime; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz); + global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); } std::filesystem::path _test_dir; @@ -1172,6 +1415,39 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_STRING); } +TEST_F(NewParquetReaderTest, RawByteArrayMappingFollowsV2ScanOption) { + write_unannotated_binary_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto string_reader = create_reader(); + const auto string_init_status = string_reader->init(&state); + ASSERT_TRUE(string_init_status.ok()) << string_init_status; + std::vector string_schema; + ASSERT_TRUE(string_reader->get_schema(&string_schema).ok()); + ASSERT_EQ(string_schema.size(), 1); + EXPECT_EQ(remove_nullable(string_schema[0].type)->get_primitive_type(), TYPE_STRING); + + RuntimeProfile binary_profile("raw_binary_mapping_profile"); + auto binary_reader = + create_reader(0, -1, &binary_profile, false, nullptr, std::nullopt, false, true); + const auto binary_init_status = binary_reader->init(&state); + ASSERT_TRUE(binary_init_status.ok()) << binary_init_status; + std::vector binary_schema; + ASSERT_TRUE(binary_reader->get_schema(&binary_schema).ok()); + ASSERT_EQ(binary_schema.size(), 1); + // The explicit VARBINARY mapping must remain available for scans whose table contract asks for it. + EXPECT_EQ(remove_nullable(binary_schema[0].type)->get_primitive_type(), TYPE_VARBINARY); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back(create_string_in_conjunct(0, {"是"})); + ASSERT_TRUE(binary_reader->open(request).ok()); + // A table-side STRING comparison may be rewritten through this VARBINARY file slot. Native + // dictionary pruning must leave the row group available for the mapping expression. + EXPECT_EQ(binary_profile.get_counter("RowGroupsFilteredByDictionary")->value(), 0); +} + // Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and // file-layer pruning hints. The reader declares those scan-request capabilities by choosing // ParquetColumnMapper itself. @@ -1203,6 +1479,10 @@ TEST_F(NewParquetReaderTest, CountComplexColumnUsesShapeOnlyPath) { EXPECT_EQ(result.count, 4); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); EXPECT_EQ(profile.get_counter("MaterializationTime")->value(), 0); + ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); + ASSERT_NE(profile.get_counter("ParsePageHeaderNum"), nullptr); + EXPECT_GT(profile.get_counter("ParsePageHeaderNum")->value(), 0); } TEST_F(NewParquetReaderTest, CountArrayColumnUsesLevelsOnlyPath) { @@ -1271,6 +1551,238 @@ TEST_F(NewParquetReaderTest, CountStructWithRepeatedChildUsesTopLevelRowBoundari } } +TEST_F(NewParquetReaderTest, NativeComplexColumnsMaterializeDirectlyAcrossBatchChanges) { + write_nullable_complex_parquet_file(_file_path); + RuntimeProfile profile("native_complex_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1), + field_projection(2)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + int batch = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + if (++batch == 1) { + // Adaptive sizing changes only the logical row cap. The persistent native readers and + // their level/string scratch must continue from the same page cursors. + reader->set_batch_size(3); + } + } + + const auto& nullable_map = assert_cast(*output[0]); + ASSERT_EQ(nullable_map.size(), ROW_COUNT); + EXPECT_FALSE(nullable_map.is_null_at(0)); + EXPECT_TRUE(nullable_map.is_null_at(1)); + EXPECT_FALSE(nullable_map.is_null_at(2)); + const auto& map = assert_cast(nullable_map.get_nested_column()); + EXPECT_EQ(map.get_offsets(), ColumnArray::Offsets64({1, 1, 1, 2, 3})); + const auto& map_keys = assert_cast(map.get_keys()); + const auto& key_values = assert_cast(map_keys.get_nested_column()); + ASSERT_EQ(key_values.size(), 3); + EXPECT_EQ(key_values.get_element(0), 10); + EXPECT_EQ(key_values.get_element(2), 30); + const auto& map_values = assert_cast(map.get_values()); + const auto& value_strings = assert_cast(map_values.get_nested_column()); + EXPECT_EQ(value_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(value_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(map_values.is_null_at(2)); + + const auto& nullable_array = assert_cast(*output[1]); + ASSERT_EQ(nullable_array.size(), ROW_COUNT); + EXPECT_TRUE(nullable_array.is_null_at(1)); + const auto& array = assert_cast(nullable_array.get_nested_column()); + EXPECT_EQ(array.get_offsets(), ColumnArray::Offsets64({2, 2, 2, 3, 4})); + const auto& array_values = assert_cast(array.get_data()); + const auto& array_strings = assert_cast(array_values.get_nested_column()); + EXPECT_EQ(array_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(array_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(array_values.is_null_at(2)); + EXPECT_EQ(array_strings.get_data_at(3).size, 4096); + + const auto& nullable_struct = assert_cast(*output[2]); + ASSERT_EQ(nullable_struct.size(), ROW_COUNT); + EXPECT_TRUE(nullable_struct.is_null_at(1)); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& payload = assert_cast(struct_column.get_column(0)); + const auto& payload_strings = assert_cast(payload.get_nested_column()); + EXPECT_EQ(payload_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(payload_strings.get_data_at(2).size, 4096); + EXPECT_TRUE(payload.is_null_at(3)); + EXPECT_EQ(payload_strings.get_data_at(4).size, 4096); + const auto& ids = assert_cast(struct_column.get_column(1)); + const auto& id_values = assert_cast(ids.get_nested_column()); + EXPECT_EQ(id_values.get_element(0), 1); + EXPECT_EQ(id_values.get_element(4), 4); + + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + ASSERT_NE(profile.get_counter("NestedBatches"), nullptr); + EXPECT_GT(profile.get_counter("NestedBatches")->value(), 0); +} + +TEST_F(NewParquetReaderTest, FullComplexChildUnderPartialParentReadsItsWholeSubtree) { + write_nested_complex_under_struct_parquet_file(_file_path); + for (size_t child_index = 0; child_index < 3; ++child_index) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + ASSERT_EQ(schema[0].children.size(), 4); + auto projection = format::LocalColumnIndex::partial_local(0); + projection.children.push_back( + format::LocalColumnIndex::local(schema[0].children[child_index].file_local_id())); + auto request = std::make_shared(); + request->non_predicate_columns = {projection}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + format::ColumnDefinition projected; + ASSERT_TRUE(format::project_column_definition(schema[0], projection, &projected).ok()); + Block block; + block.insert(ColumnWithTypeAndName(projected.type->create_column(), projected.type, + projected.name)); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_GT(rows, 0); + const auto& outer = nullable_nested_column(block, 0); + ASSERT_EQ(outer.tuple_size(), 1); + const auto& nullable_nested = assert_cast(outer.get_column(0)); + if (child_index == 0) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + ASSERT_EQ(nested.tuple_size(), 2); + const auto& payload = assert_cast(nested.get_column(0)); + EXPECT_EQ(assert_cast(payload.get_nested_column()) + .get_data_at(0) + .to_string(), + "small"); + const auto& ids = assert_cast(nested.get_column(1)); + EXPECT_EQ(ids.get_nested_column().get_int(0), 1); + } else if (child_index == 1) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 2); + EXPECT_EQ(nested.get_data().size(), 4); + } else { + const auto& nested = assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 1); + EXPECT_EQ(nested.get_keys().size(), 3); + EXPECT_EQ(nested.get_values().size(), 3); + } + } +} + +TEST_F(NewParquetReaderTest, NativeNestedMapUsesOuterKeyRepetitionShape) { + const char* source_root = std::getenv("ROOT"); + ASSERT_NE(source_root, nullptr); + _file_path = + std::string(source_root) + "/regression-test/data/external_table_p0/tvf/comp.parquet"; + ASSERT_TRUE(std::filesystem::exists(_file_path)); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + for (size_t position = 0; position < schema.size(); ++position) { + request->non_predicate_columns.push_back(field_projection(cast_set(position))); + } + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const Status status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + total_rows += rows; + } + EXPECT_GT(total_rows, 0); +} + +TEST_F(NewParquetReaderTest, NativeDecimalAndFixedBinaryMaterializeDirectly) { + write_decimal_and_fixed_binary_parquet_file(_file_path); + RuntimeProfile profile("native_decimal_fixed_binary_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + reader->set_batch_size(3); + } + + const auto& decimals = assert_cast( + assert_cast(*output[0]).get_nested_column()); + ASSERT_EQ(decimals.size(), ROW_COUNT); + EXPECT_EQ(decimals.get_element(0), Decimal128V3(1234567)); + EXPECT_EQ(decimals.get_element(1), Decimal128V3(-1)); + EXPECT_EQ(decimals.get_element(3), Decimal128V3(-987654321)); + + const auto& fixed_values = assert_cast( + assert_cast(*output[1]).get_nested_column()); + ASSERT_EQ(fixed_values.size(), ROW_COUNT); + EXPECT_EQ(fixed_values.get_data_at(0).to_string(), "ABCD"); + EXPECT_EQ(fixed_values.get_data_at(1).to_string(), std::string("\0x\0y", 4)); + EXPECT_EQ(fixed_values.get_data_at(4).to_string(), std::string("\xff\x00\x7f\x80", 4)); + + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); + EXPECT_EQ(profile.get_counter("ConvertTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { write_struct_filter_parquet_file(_file_path); auto reader = create_reader(); @@ -1297,6 +1809,35 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { EXPECT_TRUE(struct_type->get_element(1)->is_nullable()); } +TEST_F(NewParquetReaderTest, ComplexColumnDoesNotMisreportSiblingPagesAsCrossing) { + write_struct_filter_parquet_file(_file_path); + RuntimeProfile profile("new_parquet_reader_complex_page_fragments"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, 4); + // Each STRUCT child reads one page per Row Group, but no individual leaf crosses a boundary in + // either batch. The aggregate fragment count remains useful without inflating crossing batches. + EXPECT_EQ(profile.get_counter("NativePageFragments")->value(), 8); + EXPECT_EQ(profile.get_counter("PageCrossingBatches")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaMapsInt96ToTimestampTzWhenTimestampTzMappingEnabled) { write_int96_timestamp_parquet_file(_file_path); auto reader = create_reader(0, -1, nullptr, true); @@ -1532,10 +2073,78 @@ TEST_F(NewParquetReaderTest, UnknownMtimeSkipsPageCacheForMutableFile) { ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); ASSERT_NE(profile.get_counter("PageCacheWriteCount"), nullptr); - EXPECT_EQ(profile.get_counter("PageReadCount")->value(), 0); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); EXPECT_EQ(profile.get_counter("PageCacheWriteCount")->value(), 0); } +TEST_F(NewParquetReaderTest, NativeFooterCacheIdentityIncludesFilesystemAndVersion) { + const auto first = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto other_fs = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-b", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto replaced = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 11, 0, 1024, 1024, false); + EXPECT_FALSE(first.empty()); + EXPECT_NE(first, other_fs); + EXPECT_NE(first, replaced); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotCrossFilesystems) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile first_profile("native_footer_cache_nameservice_a"); + auto first = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-a", 1234567); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + RuntimeProfile second_profile("native_footer_cache_nameservice_b"); + auto second = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-b", 1234567); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheSkipsMutableUnknownVersion) { + EXPECT_TRUE(format::parquet::detail::build_native_file_cache_key("hdfs://nameservice-a", + "/warehouse/mutable.parquet", + 0, 0, 1024, 1024, false) + .empty()); + EXPECT_FALSE( + format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/immutable.parquet", 0, 0, 1024, 1024, true) + .empty()); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotReuseMutableUnknownVersion) { + _file_path = (_test_dir / "mutable_footer_cache.parquet").string(); + write_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("native_footer_cache_mutable_first"); + auto first = create_reader(0, -1, &first_profile); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + write_parquet_file(_file_path); + RuntimeProfile second_profile("native_footer_cache_mutable_second"); + auto second = create_reader(0, -1, &second_profile); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterSizeIsBoundedBeforeMetadataAllocation) { + constexpr size_t file_size = 256UL << 20; + constexpr size_t metadata_limit = 100UL << 20; + const auto status = format::parquet::detail::validate_native_footer_size( + static_cast(metadata_limit + 1), file_size, metadata_limit); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("metadata limit"), std::string::npos); +} + TEST_F(NewParquetReaderTest, UnknownMtimeUsesPageCacheForImmutableFile) { _file_path = (_test_dir / "unknown_mtime_page_cache.parquet").string(); write_parquet_file(_file_path); @@ -1645,24 +2254,42 @@ TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); ASSERT_NE(profile.get_counter("ReaderReadRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSelectRows"), nullptr); - ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterHitCache"), nullptr); ASSERT_GT(profile.get_counter("FileReaderCreateTime")->value(), 0); EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 3); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 2); + TRuntimeProfileTree profile_tree; + profile.to_thrift(&profile_tree); + ASSERT_FALSE(profile_tree.nodes.empty()); + const auto parquet_children = + profile_tree.nodes.front().child_counters_map.find("ParquetReader"); + ASSERT_NE(parquet_children, profile_tree.nodes.front().child_counters_map.end()); + EXPECT_TRUE(parquet_children->second.contains("RowsFilteredByConjunct")); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 1); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderReadRows")->value(), ROW_COUNT + 3); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 2); EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 3); - EXPECT_GT(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); EXPECT_GT(profile.get_counter("MaterializationTime")->value(), 0); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + EXPECT_EQ(profile.get_counter("FileFooterReadCalls")->value() + + profile.get_counter("FileFooterHitCache")->value(), + 1); rows = 0; eof = false; @@ -1777,14 +2404,61 @@ TEST_F(NewParquetReaderTest, EmptySelectionUpdatesProfileCounters) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 0); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 0); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 1); } +TEST_F(NewParquetReaderTest, ProfileNestsFormatReaderBelowFileReaderAndRecordsTotalTime) { + RuntimeProfile profile("new_parquet_reader_hierarchy_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + + auto* parquet_total = profile.get_counter("ParquetReader"); + ASSERT_NE(parquet_total, nullptr); + EXPECT_GT(parquet_total->value(), 0); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains(RuntimeProfile::ROOT_COUNTER)); + EXPECT_TRUE(children.at(RuntimeProfile::ROOT_COUNTER).contains("FileScannerV2")); + ASSERT_TRUE(children.contains("FileScannerV2")); + EXPECT_TRUE(children.at("FileScannerV2").contains("TableReader")); + ASSERT_TRUE(children.contains("TableReader")); + EXPECT_TRUE(children.at("TableReader").contains("FileReader")); + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + EXPECT_TRUE(children.at("FileReader").contains("ParquetReader")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("ColumnReadTime")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowGroupsReadNum")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredRowsByGroup")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredBytes")); + EXPECT_TRUE(children.at("ParquetReader").contains("FileNum")); +} + TEST_F(NewParquetReaderTest, ReadMultiPredicateColumnsBeforeExpressionFilter) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); @@ -2010,41 +2684,6 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByStatistics) { TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto value_chunk = row_group->ColumnChunk(1); - ASSERT_NE(value_chunk, nullptr); - ASSERT_TRUE(value_chunk->has_dictionary_page()); - ASSERT_TRUE(value_chunk->statistics() == nullptr || - !value_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 2); - - format::FileScanRequest plan_request; - plan_request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - plan_request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - plan_request, scan_range, false, &plan) - .ok()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 5); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 5); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, 1); - auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); @@ -2080,6 +2719,55 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { EXPECT_EQ(values, std::vector({"lm"})); } +TEST_F(NewParquetReaderTest, DictionaryPruningPublishesColdAndWarmNativePageProfile) { + const double old_cache_threshold = config::parquet_page_cache_decompress_threshold; + config::parquet_page_cache_decompress_threshold = 100.0; + Defer restore_cache_threshold { + [&] { config::parquet_page_cache_decompress_threshold = old_cache_threshold; }}; + write_dictionary_filter_parquet_file(_file_path, ::parquet::Compression::SNAPPY); + + auto open_pruned_reader = [&](RuntimeProfile* profile) { + auto reader = create_reader(0, -1, profile, false, nullptr, std::nullopt, true); + TQueryOptions query_options; + query_options.__set_enable_parquet_file_page_cache(true); + RuntimeState state {query_options, TQueryGlobals()}; + RETURN_IF_ERROR(reader->init(&state)); + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"not-present"})); + use_schema_order_positions(request.get(), schema); + RETURN_IF_ERROR(reader->open(request)); + EXPECT_EQ(profile->get_counter("RowGroupsFilteredByDictionary")->value(), 0); + EXPECT_EQ(profile->get_counter("PageReadCount")->value(), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + // Expensive dictionary probes are current-row-group work now, so advance the scheduler + // once before inspecting page-cache and pruning counters. + return reader->get_block(&block, &rows, &eof); + }; + + RuntimeProfile cold_profile("dictionary_pruning_cold_profile"); + ASSERT_TRUE(open_pruned_reader(&cold_profile).ok()); + for (const auto* counter_name : + {"RowGroupsFilteredByDictionary", "PageReadCount", "PageCacheWriteCount", + "ParsePageHeaderNum", "DecompressCount", "DecodeDictTime"}) { + ASSERT_NE(cold_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(cold_profile.get_counter(counter_name)->value(), 0) << counter_name; + } + + RuntimeProfile warm_profile("dictionary_pruning_warm_profile"); + ASSERT_TRUE(open_pruned_reader(&warm_profile).ok()); + for (const auto* counter_name : {"RowGroupsFilteredByDictionary", "PageReadCount", + "PageCacheHitCount", "ParsePageHeaderNum", "DecodeDictTime"}) { + ASSERT_NE(warm_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(warm_profile.get_counter(counter_name)->value(), 0) << counter_name; + } +} + TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { write_single_row_group_dictionary_filter_parquet_file(_file_path); auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); @@ -2135,7 +2823,102 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { EXPECT_GE(profile.get_counter("ReaderSelectRows")->value(), 8); } -TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) { +TEST_F(NewParquetReaderTest, FixedWidthDictionaryPredicateFiltersRowsByDictionaryId) { + write_fixed_width_dictionary_filter_parquet_file(_file_path); + + RuntimeProfile profile("new_parquet_reader_fixed_width_dictionary_filter_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_int32_dictionary_equals_conjunct(1, 20)); + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + if (rows == 0) { + continue; + } + const auto& id_column = nullable_nested_column(block, 0); + const auto& value_column = nullable_nested_column(block, 1); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + values.push_back(value_column.get_element(row)); + } + } + + EXPECT_EQ(ids, std::vector({2, 4, 6})); + EXPECT_EQ(values, std::vector({20, 20, 20})); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 3); + EXPECT_EQ(profile.get_counter("DictFilterCandidateColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterColumns")->value(), 1); + EXPECT_EQ(profile.get_counter("DictFilterUnsupportedColumns")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterReadFailures")->value(), 0); +} + +TEST_F(NewParquetReaderTest, AllFixedWidthDictionaryTypesDecodeThroughDictionaryIds) { + write_all_fixed_width_dictionary_filter_parquet_file(_file_path); + + auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); + auto row_group = parquet_file_reader->metadata()->RowGroup(0); + ASSERT_NE(row_group, nullptr); + ASSERT_EQ(row_group->num_columns(), 6); + const std::array<::parquet::Type::type, 5> expected_types { + ::parquet::Type::INT64, ::parquet::Type::FLOAT, ::parquet::Type::DOUBLE, + ::parquet::Type::FIXED_LEN_BYTE_ARRAY, ::parquet::Type::INT96}; + for (int column = 1; column < row_group->num_columns(); ++column) { + ASSERT_TRUE(row_group->ColumnChunk(column)->has_dictionary_page()) << column; + EXPECT_EQ(row_group->ColumnChunk(column)->type(), expected_types[column - 1]) << column; + } + + RuntimeProfile profile("new_parquet_reader_all_fixed_width_dictionary_filter_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 6); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + for (int column = 1; column < 6; ++column) { + request->predicate_columns.push_back(field_projection(column)); + request->conjuncts.push_back(create_dictionary_accept_all_conjunct(column)); + } + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + total_rows += rows; + } + + EXPECT_EQ(total_rows, 6); + EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterCandidateColumns")->value(), 5); + EXPECT_EQ(profile.get_counter("DictFilterColumns")->value(), 5); + EXPECT_EQ(profile.get_counter("DictFilterUnsupportedColumns")->value(), 0); + EXPECT_EQ(profile.get_counter("DictFilterReadFailures")->value(), 0); +} + +TEST_F(NewParquetReaderTest, DictionaryPredicateReaderIsSharedOutsideMergeRangeReader) { write_dictionary_filter_with_trailing_column_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_dictionary_filter_merge_profile"); @@ -2177,8 +2960,14 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) EXPECT_EQ(values, std::vector({"az", "za"})); EXPECT_EQ(payloads, std::vector({20, 50})); EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); + // The native dictionary probe keeps its reader and cursor for predicate data pages. Other + // projected chunks still use the row-group merge reader, so probing never duplicates the + // dictionary read or perturbs the merge range's sequential access order. ASSERT_NE(profile.get_counter("MergedIO"), nullptr); ASSERT_NE(profile.get_counter("MergedBytes"), nullptr); + EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); } TEST_F(NewParquetReaderTest, DictionaryPredicateWorksWithoutRuntimeProfile) { @@ -2258,7 +3047,9 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateSkipsRemainingPredicateColumnsWh // second predicate column is skipped after the selection becomes empty, which verifies the // StarRocks-style round-by-round policy: only rows surviving previous predicates are read. EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 6); - EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 6); + // Five dictionary ids are rejected before the residual predicate; the remaining predicate + // reader then skips all six logical rows once the selection is empty. + EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 11); } TEST_F(NewParquetReaderTest, DictionaryPredicateRunsResidualConjunctOnSurvivors) { @@ -2346,206 +3137,11 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateKeepsNestedOrResidualConjunct) { EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 1); } -TEST_F(NewParquetReaderTest, ScanRangeFiltersRowGroupsBeforeDictionaryPruning) { - write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 2); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByStatistics) { - write_struct_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 2); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByDictionary) { - write_nested_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto name_chunk = row_group->ColumnChunk(1); - ASSERT_NE(name_chunk, nullptr); - ASSERT_TRUE(name_chunk->has_dictionary_page()); - ASSERT_TRUE(name_chunk->statistics() == nullptr || !name_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[1]->name, "name"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 6); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, PlannerNarrowsRowRangesByPageIndex) { - write_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - - format::FileScanRequest request; - request.predicate_columns = {field_projection(0)}; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_GT(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_LT(plan.row_groups[0].selected_ranges.front().length, 128); - auto skip_plan_it = plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(skip_plan_it, plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(skip_plan_it->second.leaf_column_id, 0); - EXPECT_GT(skip_plan_it->second.skipped_ranges.size(), 0); - EXPECT_GT(skip_plan_it->second.skipped_pages.size(), 1); - ASSERT_EQ(skip_plan_it->second.skipped_pages.size(), - skip_plan_it->second.skipped_page_compressed_sizes.size()); - int64_t skipped_compressed_bytes = 0; - for (size_t page_idx = 0; page_idx < skip_plan_it->second.skipped_pages.size(); ++page_idx) { - if (skip_plan_it->second.should_skip_page(page_idx)) { - skipped_compressed_bytes += skip_plan_it->second.skipped_page_compressed_size(page_idx); - } - } - EXPECT_GT(skipped_compressed_bytes, 0); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_GT(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotNarrowRowRangesByPageIndex) { - write_nested_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().length, - parquet_file_reader->metadata()->RowGroup(0)->num_rows()); - EXPECT_TRUE(plan.row_groups[0].page_skip_plans.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - -TEST_F(NewParquetReaderTest, PageIndexFilteredPagesDoNotDoubleSkipOutputColumns) { +// Scenario: the selected range starts after page-index-pruned rows. The scheduler defers that range +// gap for the non-predicate payload reader, then flushes it exactly once before materialization. The +// native RowRanges/OffsetIndex plan advances both readers without decoding the rejected pages or +// double-skipping row 64. +TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { write_page_index_filter_pair_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_page_skip"); auto reader = create_reader(0, -1, &profile); @@ -2568,7 +3164,8 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredPagesDoNotDoubleSkipOutputColumns) bool eof = false; while (!eof) { size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } @@ -2586,6 +3183,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredPagesDoNotDoubleSkipOutputColumns) ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RangeGapSkippedRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); ASSERT_NE(profile.get_counter("RowGroupFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexReadTime"), nullptr); @@ -2595,6 +3193,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredPagesDoNotDoubleSkipOutputColumns) EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 64); EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); EXPECT_GT(profile.get_counter("RowGroupFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexReadTime")->value(), 0); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 04d46097c98648..9bdb0c7562f0b6 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -21,10 +21,13 @@ #include #include #include +#include #include +#include #include #include +#include #include #include #include @@ -36,17 +39,23 @@ #include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_array.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/field.h" +#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" +#include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_reader.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "gen_cpp/PlanNodes_types.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -54,6 +63,8 @@ #include "storage/index/zone_map/zonemap_eval_context.h" #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/utils.h" +#include "util/coding.h" +#include "util/thrift_util.h" namespace doris { namespace { @@ -69,6 +80,13 @@ const ColumnInt32& int32_data_column(const IColumn& column) { return assert_cast(column); } +const ColumnInt64& int64_data_column(const IColumn& column) { + if (const auto* nullable_column = check_and_get_column(&column)) { + return assert_cast(nullable_column->get_nested_column()); + } + return assert_cast(column); +} + const ColumnString& string_data_column(const IColumn& column) { if (const auto* nullable_column = check_and_get_column(&column)) { return assert_cast(nullable_column->get_nested_column()); @@ -76,6 +94,64 @@ const ColumnString& string_data_column(const IColumn& column) { return assert_cast(column); } +TEST(ParquetScanMetadataSafetyTest, CheckedChunkRangesDrivePrefetchAndSplitAssignment) { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + metadata.__set_schema({root, leaf}); + + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + column.__set_data_page_offset(100); + column.__set_dictionary_page_offset(-1); + column.__set_total_compressed_size(20); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + row_group.__set_columns({chunk}); + metadata.__set_row_groups({row_group}); + + auto schema = std::make_unique(); + schema->local_id = 0; + schema->leaf_column_id = 0; + schema->type = std::make_shared(); + std::vector> file_schema; + file_schema.push_back(std::move(schema)); + std::vector ranges; + ASSERT_TRUE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges[0].offset, 100); + EXPECT_EQ(ranges[0].size, 20); + + std::vector first_rows; + std::vector selected; + format::parquet::ParquetScanRange first_split { + .start_offset = 0, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, first_split, &first_rows, &selected) + .ok()); + EXPECT_TRUE(selected.empty()); + format::parquet::ParquetScanRange second_split { + .start_offset = 100, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, second_split, &first_rows, &selected) + .ok()); + EXPECT_EQ(selected, std::vector({0})); + + metadata.row_groups[0].columns[0].meta_data.__set_data_page_offset(190); + EXPECT_FALSE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); +} + class Int32ZoneMapExpr final : public VExpr { public: enum class Op { GE, GT, LT }; @@ -191,17 +267,223 @@ class Int32PairSumExpr final : public VExpr { const std::string _expr_name = "Int32PairSumExpr"; }; +class Int32DirectGreaterExpr final : public VExpr { +public: + Int32DirectGreaterExpr(int column_id, int32_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int32_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_INT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int32_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int32_t _lower_bound; + const std::string _expr_name = "Int32DirectGreaterExpr"; +}; + +class Int64DirectGreaterExpr final : public VExpr { +public: + Int64DirectGreaterExpr(int column_id, int64_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int64_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_BIGINT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + if (value_width != sizeof(int64_t)) { + return Status::Corruption("BIGINT raw predicate received {}-byte values", value_width); + } + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int64_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int64_t _lower_bound; + const std::string _expr_name = "Int64DirectGreaterExpr"; +}; + +class AlwaysTrueSingleColumnExpr final : public VExpr { +public: + explicit AlwaysTrueSingleColumnExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + const std::string _expr_name = "AlwaysTrueSingleColumnExpr"; +}; + VExprContextSPtr create_int32_zonemap_conjunct(int column_id, Int32ZoneMapExpr::Op op, int32_t value) { return VExprContext::create_shared(std::make_shared(column_id, op, value)); } +VExprContextSPtr create_int32_function_conjunct(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t value) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto result_type = make_nullable(std::make_shared()); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({nullable_int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(result_type->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(result_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(true); + auto root = VectorizedFnCall::create_shared(node); + root->add_child( + VSlotRef::create_shared(column_id, column_id, -1, nullable_int_type, "plain_id")); + root->add_child(VLiteral::create_shared(int_type, Field::create_field(value))); + auto context = VExprContext::create_shared(std::move(root)); + // Direct evaluation does not execute the expression, but a fallback must still fail this test + // instead of silently using an unprepared test-only context. + context->_prepared = true; + context->_opened = true; + return context; +} + +VExprContextSPtr create_int32_mod_greater_than_conjunct(int column_id) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto nullable_bool_type = make_nullable(std::make_shared()); + auto function_node = [](const std::string& name, TExprNodeType::type node_type, + TExprOpcode::type opcode, const DataTypePtr& return_type, + const std::vector& argument_types) { + TFunctionName function_name; + function_name.__set_function_name(name); + TFunction function; + function.__set_name(function_name); + function.__set_binary_type(TFunctionBinaryType::BUILTIN); + std::vector thrift_argument_types; + for (const auto& argument_type : argument_types) { + thrift_argument_types.push_back(argument_type->to_thrift()); + } + function.__set_arg_types(thrift_argument_types); + function.__set_ret_type(return_type->to_thrift()); + function.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(node_type); + node.__set_opcode(opcode); + node.__set_type(return_type->to_thrift()); + node.__set_fn(function); + node.__set_num_children(static_cast(argument_types.size())); + node.__set_is_nullable(return_type->is_nullable()); + return node; + }; + + auto modulo = VectorizedFnCall::create_shared( + function_node("mod", TExprNodeType::ARITHMETIC_EXPR, TExprOpcode::MOD, + nullable_int_type, {nullable_int_type, int_type})); + modulo->add_child( + VSlotRef::create_shared(column_id, column_id, -1, nullable_int_type, "plain_id")); + modulo->add_child(VLiteral::create_shared(int_type, Field::create_field(-1))); + + auto greater_than = VectorizedFnCall::create_shared( + function_node("gt", TExprNodeType::BINARY_PRED, TExprOpcode::GT, nullable_bool_type, + {nullable_int_type, int_type})); + greater_than->add_child(std::move(modulo)); + greater_than->add_child(VLiteral::create_shared(int_type, Field::create_field(0))); + return VExprContext::create_shared(std::move(greater_than)); +} + VExprContextSPtr create_int32_pair_sum_conjunct(int left_column_id, int right_column_id, int32_t upper_bound) { return VExprContext::create_shared( std::make_shared(left_column_id, right_column_id, upper_bound)); } +VExprContextSPtr create_int32_direct_greater_conjunct(int column_id, int32_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + +VExprContextSPtr create_int64_direct_greater_conjunct(int column_id, int64_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + +VExprContextSPtr create_always_true_single_column_conjunct(int column_id) { + return VExprContext::create_shared(std::make_shared(column_id)); +} + int64_t counter_value(RuntimeProfile& profile, const std::string& name) { auto* counter = profile.get_counter(name); DORIS_CHECK(counter != nullptr); @@ -222,6 +504,14 @@ std::shared_ptr build_int32_array(const std::vector& valu return finish_array(&builder); } +std::shared_ptr build_uint32_array(const std::vector& values) { + arrow::UInt32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -277,7 +567,8 @@ std::shared_ptr build_list_array() { void write_table(const std::string& file_path, const std::shared_ptr& table, int64_t row_group_size, bool enable_dictionary = false, - bool enable_page_index = false, bool enable_statistics = true) { + bool enable_page_index = false, bool enable_statistics = true, + std::optional<::parquet::Encoding::type> encoding = std::nullopt) { auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); std::shared_ptr out = *file_result; @@ -291,6 +582,9 @@ void write_table(const std::string& file_path, const std::shared_ptr out = *file_result; + + // Arrow intentionally writes time32 as local time (isAdjustedToUTC=false), so use Parquet's + // low-level writer to produce the unsupported required logical type needed by this aggregate + // regression. Required is important: Nereids may push COUNT(col) because NULL filtering cannot + // change its value, which is precisely the path where an empty aggregate request lost `col`. + const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( + "unsupported_time", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), + ::parquet::Type::INT32); + const auto schema_node = ::parquet::schema::GroupNode::Make( + "schema", ::parquet::Repetition::REQUIRED, {adjusted_time}); + const auto schema = std::static_pointer_cast<::parquet::schema::GroupNode>(schema_node); + auto writer = ::parquet::ParquetFileWriter::Open(out, schema); + auto* row_group = writer->AppendRowGroup(); + auto* time_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + const int32_t values[] = {1000, 2000}; + EXPECT_EQ(time_writer->WriteBatch(2, nullptr, nullptr, values), 2); + time_writer->Close(); + row_group->Close(); + writer->Close(); +} + void write_int_pair_parquet_file(const std::string& file_path, int64_t row_group_size = 2, - bool enable_statistics = true) { + bool enable_statistics = true, + std::optional<::parquet::Encoding::type> encoding = std::nullopt) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("score", arrow::int32(), false), }); auto table = arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), build_int32_array({10, 20, 30, 40, 50, 60})}); - write_table(file_path, table, row_group_size, false, false, enable_statistics); + write_table(file_path, table, row_group_size, false, false, enable_statistics, encoding); +} + +void write_uint32_pair_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("id", arrow::uint32(), false), + arrow::field("score", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_uint32_array({1, 2147483648U, 4294957294U}), + build_int32_array({10, 20, 30})}); + write_table(file_path, table, 3, false, false, false); +} + +std::vector serialize_test_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +void write_misdeclared_two_page_parquet_file(const std::string& file_path) { + const std::array first_values {1, 2}; + std::vector first_payload(sizeof(first_values)); + memcpy(first_payload.data(), first_values.data(), first_payload.size()); + tparquet::PageHeader first_header; + first_header.type = tparquet::PageType::DATA_PAGE_V2; + first_header.__set_compressed_page_size(first_payload.size()); + first_header.__set_uncompressed_page_size(first_payload.size()); + first_header.__isset.data_page_header_v2 = true; + first_header.data_page_header_v2.__set_num_values(first_values.size()); + first_header.data_page_header_v2.__set_num_nulls(0); + first_header.data_page_header_v2.__set_num_rows(first_values.size()); + first_header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + first_header.data_page_header_v2.__set_definition_levels_byte_length(0); + first_header.data_page_header_v2.__set_repetition_levels_byte_length(0); + first_header.data_page_header_v2.__set_is_compressed(false); + auto column_bytes = serialize_test_page(first_header, first_payload); + + // Bit width 1 followed by an RLE run of two dictionary ids. The payload is structurally valid + // so the raw path reaches its late-encoding guard instead of failing while loading the page. + const std::vector second_payload {1, 4, 0}; + tparquet::PageHeader second_header = first_header; + second_header.__set_compressed_page_size(second_payload.size()); + second_header.__set_uncompressed_page_size(second_payload.size()); + second_header.data_page_header_v2.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + auto second_page = serialize_test_page(second_header, second_payload); + column_bytes.insert(column_bytes.end(), second_page.begin(), second_page.end()); + + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("id"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + // Deliberately omit the second page's unsupported encoding to emulate untrusted footer + // metadata. The reader must reject it before attempting a typed fallback after partial cursor + // progress. + column.__set_encodings({tparquet::Encoding::PLAIN, tparquet::Encoding::RLE}); + column.__set_path_in_schema({"id"}); + column.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column.__set_num_values(4); + column.__set_total_uncompressed_size(column_bytes.size()); + column.__set_total_compressed_size(column_bytes.size()); + column.__set_data_page_offset(4); + tparquet::ColumnChunk chunk; + chunk.__set_file_offset(4); + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(column_bytes.size()); + row_group.__set_num_rows(4); + tparquet::FileMetaData metadata; + metadata.__set_version(2); + metadata.__set_schema({root, leaf}); + metadata.__set_num_rows(4); + metadata.__set_row_groups({row_group}); + + std::vector footer; + ThriftSerializer serializer(/*compact=*/true, 1024); + DORIS_CHECK(serializer.serialize(&metadata, &footer).ok()); + std::ofstream output(file_path, std::ios::binary | std::ios::trunc); + output.write("PAR1", 4); + output.write(reinterpret_cast(column_bytes.data()), column_bytes.size()); + output.write(reinterpret_cast(footer.data()), footer.size()); + std::array footer_size {}; + encode_fixed32_le(footer_size.data(), cast_set(footer.size())); + output.write(reinterpret_cast(footer_size.data()), footer_size.size()); + output.write("PAR1", 4); + output.close(); + DORIS_CHECK(output.good()); +} + +void write_long_prefix_parquet_file(const std::string& file_path, size_t rows) { + std::vector ids(rows); + std::vector first(rows); + std::vector second(rows); + std::iota(ids.begin(), ids.end(), 0); + for (size_t row = 0; row < rows; ++row) { + first[row] = static_cast(row + 10); + second[row] = static_cast(row + 20); + } + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("first", arrow::int32(), false), + arrow::field("second", arrow::int32(), false)}); + auto table = arrow::Table::Make( + schema, {build_int32_array(ids), build_int32_array(first), build_int32_array(second)}); + write_table(file_path, table, rows, false, false, false); } void write_binary_minmax_parquet_file(const std::string& file_path) { @@ -343,27 +773,33 @@ void write_list_parquet_file(const std::string& file_path) { write_table(file_path, table, 2); } -void write_page_index_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); +void write_int_list_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), + arrow::field("xs", arrow::list(arrow::int32()), false), }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); - write_table(file_path, table, ids.size(), false, true); + auto table = arrow::Table::Make(schema, {build_int32_array({1, 2, 3}), build_list_array()}); + write_table(file_path, table, 3); } -void write_page_index_pair_parquet_file(const std::string& file_path) { +void write_page_index_parquet_file(const std::string& file_path) { std::vector ids(128); std::iota(ids.begin(), ids.end(), 0); auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), - arrow::field("score", arrow::int32(), false), }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(ids)}); + auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); write_table(file_path, table, ids.size(), false, true); } +void write_multi_row_group_page_index_parquet_file(const std::string& file_path) { + std::vector ids(384); + std::iota(ids.begin(), ids.end(), 0); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + write_table(file_path, table, 128, false, true); +} + int64_t parquet_column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { return column_metadata.has_dictionary_page() ? static_cast(column_metadata.dictionary_page_offset()) @@ -409,24 +845,6 @@ void use_schema_order_positions(format::FileScanRequest* request, } } -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - auto schema_descriptor = reader.metadata()->schema(); - EXPECT_NE(schema_descriptor, nullptr); - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - return file_schema; -} - -int64_t count_range_rows(const std::vector& ranges) { - int64_t rows = 0; - for (const auto& range : ranges) { - rows += range.length; - } - return rows; -} - class ParquetScanTest : public testing::Test { protected: void SetUp() override { @@ -483,87 +901,76 @@ TEST(ParquetScanSelectionTest, CompactFilterShrinksCurrentSelection) { EXPECT_TRUE(selection.verify(selected_rows, 6).ok()); } -TEST_F(ParquetScanTest, PlanRowGroupsAppliesScanRangeBeforeStatistics) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 1); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 3); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 1); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 2); +TEST(ParquetScanAdaptivePredicateTest, OrdersByObservedCostPerRejectedRow) { + using format::parquet::detail::AdaptivePredicateStats; + std::unordered_map stats; + stats.emplace(0, AdaptivePredicateStats { + .cost_per_input_row_ns = 10, .survival_ratio = 0.8, .samples = 3}); + stats.emplace(1, AdaptivePredicateStats { + .cost_per_input_row_ns = 20, .survival_ratio = 0.1, .samples = 3}); + stats.emplace(2, AdaptivePredicateStats { + .cost_per_input_row_ns = 50, .survival_ratio = 0.5, .samples = 3}); + + const auto order = format::parquet::detail::order_adaptive_predicates({0, 1, 2}, stats); + EXPECT_EQ(order, std::vector({1, 0, 2})); + const auto prefetched = format::parquet::detail::adaptive_prefetch_prefix(order, stats, 0.25); + EXPECT_EQ(prefetched, std::vector({1})); } -TEST_F(ParquetScanTest, PlanRowGroupsPreservesFirstFileRowAcrossPrunedRowGroups) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); +TEST(ParquetScanAdaptivePredicateTest, SamplesWarmupThenAtLowFrequency) { + using format::parquet::detail::should_sample_adaptive_predicate; + for (size_t samples = 0; samples < 8; ++samples) { + EXPECT_TRUE(should_sample_adaptive_predicate(samples, samples)); + } + EXPECT_FALSE(should_sample_adaptive_predicate(8, 9)); + EXPECT_TRUE(should_sample_adaptive_predicate(8, 16)); + EXPECT_FALSE(should_sample_adaptive_predicate(9, 17)); + EXPECT_TRUE(should_sample_adaptive_predicate(9, 32)); +} - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); +TEST(ParquetScanDeleteConjunctTest, RejectsInputColumnAsEphemeralResult) { + EXPECT_TRUE(format::parquet::detail::validate_ephemeral_expr_result_column(2, 0, 2) + .is()); + EXPECT_TRUE(format::parquet::detail::validate_ephemeral_expr_result_column(2, 2, 3).ok()); + EXPECT_TRUE(format::parquet::detail::validate_ephemeral_expr_result_column(2, 3, 3) + .is()); +} - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.row_groups[0].first_file_row, 4); - EXPECT_EQ(plan.row_groups[0].row_group_rows, 2); - ASSERT_EQ(plan.row_groups[0].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].length, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 2); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 4); -} - -TEST_F(ParquetScanTest, PlanRowGroupsSelectsAllRowGroupsWithoutFilters) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); +TEST(ParquetScanAdaptivePredicateTest, ThrowingNestedFunctionDisablesSelectedRowReordering) { + using format::parquet::detail::AdaptivePredicateStats; + std::unordered_map first_batch_stats; + first_batch_stats.emplace( + 0, AdaptivePredicateStats { + .cost_per_input_row_ns = 20, .survival_ratio = 0.99, .samples = 1}); + first_batch_stats.emplace( + 1, AdaptivePredicateStats { + .cost_per_input_row_ns = 1, .survival_ratio = 0.1, .samples = 1}); + EXPECT_EQ(format::parquet::detail::order_adaptive_predicates({0, 1}, first_batch_stats), + (std::vector {1, 0})); + + const auto total_comparison = create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0); + const auto throwing_comparison = create_int32_mod_greater_than_conjunct(0); + EXPECT_TRUE(total_comparison->root()->is_safe_to_execute_on_selected_rows()); + // The second-batch order above can reject MIN_INT before mod(MIN_INT, -1) runs. The nested + // partial function must therefore keep the request on full-batch, error-preserving execution. + EXPECT_FALSE(throwing_comparison->root()->is_safe_to_execute_on_selected_rows()); +} - format::FileScanRequest request; - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); +TEST(ParquetScanSmallFileTest, StagesOnlyBoundedHttpObjects) { + using format::parquet::detail::should_stage_small_http_file; + EXPECT_TRUE(should_stage_small_http_file("http://host/tiny.parquet", 512, 1024)); + EXPECT_TRUE(should_stage_small_http_file("https://host/tiny.parquet", 1024, 1024)); + EXPECT_FALSE(should_stage_small_http_file("https://host/large.parquet", 1025, 1024)); + EXPECT_FALSE(should_stage_small_http_file("/tmp/tiny.parquet", 512, 1024)); + EXPECT_FALSE(should_stage_small_http_file("s3://bucket/tiny.parquet", 512, 1024)); +} - ASSERT_EQ(plan.row_groups.size(), 3); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 3); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 3); - for (size_t row_group_idx = 0; row_group_idx < plan.row_groups.size(); ++row_group_idx) { - EXPECT_EQ(plan.row_groups[row_group_idx].row_group_id, row_group_idx); - EXPECT_EQ(plan.row_groups[row_group_idx].first_file_row, - static_cast(row_group_idx * 2)); - ASSERT_EQ(plan.row_groups[row_group_idx].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].length, 2); - EXPECT_TRUE(plan.row_groups[row_group_idx].page_skip_plans.empty()); - } +TEST(ParquetScanSelectionTest, NativeLazySkipBitmapIsBounded) { + using format::parquet::detail::MAX_NATIVE_LAZY_SKIP_ROWS; + using format::parquet::detail::bounded_native_lazy_skip_rows; + EXPECT_EQ(bounded_native_lazy_skip_rows(1), 1); + EXPECT_EQ(bounded_native_lazy_skip_rows(MAX_NATIVE_LAZY_SKIP_ROWS + 1), + MAX_NATIVE_LAZY_SKIP_ROWS); } TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater) { @@ -573,7 +980,8 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater .first_file_row = ConditionCacheContext::GRANULE_SIZE, .row_group_rows = ConditionCacheContext::GRANULE_SIZE, .selected_ranges = {{.start = 0, .length = ConditionCacheContext::GRANULE_SIZE}}, - .page_skip_plans = {}}); + .page_skip_plans = {}, + .offset_indexes = {}}); format::parquet::ParquetScanScheduler scheduler; scheduler.set_plan(std::move(plan)); @@ -588,131 +996,6 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater EXPECT_EQ(ctx->base_granule, 0); } -TEST_F(ParquetScanTest, PageIndexIntersectsMultipleFiltersAndBuildsSkipPlan) { - write_page_index_pair_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest single_filter_request; - format::FileScanRequestBuilder single_filter_builder(&single_filter_request); - ASSERT_TRUE(single_filter_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - single_filter_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - format::parquet::RowGroupScanPlan single_filter_plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - single_filter_request, scan_range, false, &single_filter_plan) - .ok()); - ASSERT_EQ(single_filter_plan.row_groups.size(), 1); - const int64_t single_filter_rows = - count_range_rows(single_filter_plan.row_groups[0].selected_ranges); - - format::FileScanRequest intersect_request; - format::FileScanRequestBuilder intersect_builder(&intersect_request); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(1)).ok()); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(1, Int32ZoneMapExpr::Op::LT, 96)); - format::parquet::RowGroupScanPlan intersect_plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - intersect_request, scan_range, false, &intersect_plan) - .ok()); - ASSERT_EQ(intersect_plan.row_groups.size(), 1); - ASSERT_FALSE(intersect_plan.row_groups[0].selected_ranges.empty()); - const int64_t intersect_rows = count_range_rows(intersect_plan.row_groups[0].selected_ranges); - EXPECT_GT(single_filter_rows, intersect_rows); - EXPECT_GT(intersect_plan.row_groups[0].selected_ranges.front().start, 0); - const auto& last_range = intersect_plan.row_groups[0].selected_ranges.back(); - EXPECT_LT(last_range.start + last_range.length, 128); - EXPECT_GT(intersect_plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(intersect_plan.pruning_stats.selected_row_ranges, - intersect_plan.row_groups[0].selected_ranges.size()); - - auto id_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(id_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(id_skip_plan->second.leaf_column_id, 0); - EXPECT_FALSE(id_skip_plan->second.empty()); - auto score_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(1); - ASSERT_NE(score_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(score_skip_plan->second.leaf_column_id, 1); - EXPECT_FALSE(score_skip_plan->second.empty()); -} - -TEST_F(ParquetScanTest, PageIndexCanFullyFilterRowGroupAfterRangeIntersection) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::LT, 32)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 1); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 128); -} - -TEST_F(ParquetScanTest, PageIndexFullRangeWhenDisabledOrUnavailable) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); - - const bool old_enable_page_index = config::enable_parquet_page_index; - config::enable_parquet_page_index = false; - std::vector selected_ranges; - std::map page_skip_plans; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - parquet_file_reader.get(), file_schema, request, 0, 128, &selected_ranges, - &page_skip_plans, &pruning_stats) - .ok()); - config::enable_parquet_page_index = old_enable_page_index; - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 128); - EXPECT_TRUE(page_skip_plans.empty()); - EXPECT_EQ(pruning_stats.page_index_read_calls, 0); - - write_int_pair_parquet_file(_file_path, 6); - auto no_index_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto no_index_schema = build_file_schema(*no_index_reader); - format::FileScanRequest no_index_request; - no_index_request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - no_index_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 3)); - selected_ranges.clear(); - page_skip_plans.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - no_index_reader.get(), no_index_schema, no_index_request, 0, 6, - &selected_ranges, &page_skip_plans, &pruning_stats) - .ok()); - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 6); - EXPECT_TRUE(page_skip_plans.empty()); -} - TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); @@ -727,6 +1010,15 @@ TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { EXPECT_EQ(count_result.count, 6); EXPECT_TRUE(count_result.columns.empty()); + format::FileAggregateResult required_count_result; + format::FileAggregateRequest required_count_request; + required_count_request.agg_type = TPushAggOp::COUNT; + required_count_request.columns.push_back({.projection = field_projection(0)}); + ASSERT_TRUE(reader->get_aggregate_result(required_count_request, &required_count_result).ok()); + // The required scalar projection is retained for logical-type validation, but after that + // validation COUNT(id) can reuse the same selected-row-group footer count as COUNT(*). + EXPECT_EQ(required_count_result.count, 6); + format::FileAggregateResult minmax_result; format::FileAggregateRequest minmax_request; minmax_request.agg_type = TPushAggOp::MINMAX; @@ -743,6 +1035,96 @@ TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { EXPECT_EQ(minmax_result.columns[1].max_value.get(), 60); } +TEST_F(ParquetScanTest, AggregateCountRejectsRequiredUnsupportedScalarProjection) { + write_required_adjusted_time_parquet_file(_file_path); + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + open_all_row_groups(reader.get()); + + format::FileAggregateRequest request; + request.agg_type = TPushAggOp::COUNT; + request.columns.push_back({.projection = field_projection(0)}); + format::FileAggregateResult result; + const auto status = reader->get_aggregate_result(request, &result); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), + std::string::npos); +} + +TEST_F(ParquetScanTest, CountStarIgnoresUnsupportedPlannerPlaceholder) { + write_required_adjusted_time_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + // A normal projection still requests the TIME_MILLIS value and must fail at open, before + // row-group pruning or physical INT32 statistics can hide the unsupported logical type. + auto projected_reader = create_reader(); + ASSERT_TRUE(projected_reader->init(&state).ok()); + auto projected_request = std::make_shared(); + format::FileScanRequestBuilder projected_builder(projected_request.get()); + ASSERT_TRUE(projected_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + const auto projected_status = projected_reader->open(projected_request); + EXPECT_TRUE(projected_status.is()) << projected_status; + + // Metadata COUNT(*) carries the same retained scan slot, but its aggregate request has no + // semantic column. The placeholder must not make open fail before footer row counts are used. + auto metadata_count_reader = create_reader(); + ASSERT_TRUE(metadata_count_reader->init(&state).ok()); + auto metadata_count_request = std::make_shared(); + format::FileScanRequestBuilder metadata_count_builder(metadata_count_request.get()); + ASSERT_TRUE(metadata_count_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + metadata_count_request->count_star_placeholder_columns.push_back(format::LocalColumnId(0)); + ASSERT_TRUE(metadata_count_reader->open(metadata_count_request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::COUNT; + format::FileAggregateResult aggregate_result; + ASSERT_TRUE( + metadata_count_reader->get_aggregate_result(aggregate_request, &aggregate_result).ok()); + EXPECT_EQ(aggregate_result.count, 2); + + // The marker is independent of aggregate eligibility. Simulate a position delete, + // which disables metadata COUNT and requires the reader to produce surviving rows. Parquet + // reads only the virtual row position, filters row 1, and fills a default placeholder for row 0 + // without validating or decoding either unsupported TIME_MILLIS value. + auto count_star_reader = create_reader(); + ASSERT_TRUE(count_star_reader->init(&state).ok()); + auto count_star_request = std::make_shared(); + format::FileScanRequestBuilder count_star_builder(count_star_request.get()); + ASSERT_TRUE(count_star_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(count_star_builder + .add_predicate_column(format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) + .ok()); + count_star_request->count_star_placeholder_columns.push_back(format::LocalColumnId(0)); + + const std::vector deleted_rows {1}; + auto delete_predicate = std::make_shared(deleted_rows); + const auto row_position = count_star_request->local_positions.at( + format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)); + delete_predicate->add_child(VSlotRef::create_shared( + cast_set(row_position.value()), cast_set(row_position.value()), -1, + std::make_shared(), format::ROW_POSITION_COLUMN_NAME)); + auto delete_context = VExprContext::create_shared(std::move(delete_predicate)); + ASSERT_TRUE(delete_context->prepare(&state, RowDescriptor()).ok()); + ASSERT_TRUE(delete_context->open(&state).ok()); + count_star_request->delete_conjuncts.push_back(std::move(delete_context)); + ASSERT_TRUE(count_star_reader->open(count_star_request).ok()); + + std::vector file_schema; + ASSERT_TRUE(count_star_reader->get_schema(&file_schema).ok()); + file_schema.push_back(format::row_position_column_definition()); + auto block = build_file_block(file_schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(count_star_reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 1); + EXPECT_EQ(block.get_by_position(0).column->size(), 1); + const auto& positions = + assert_cast(*block.get_by_position(row_position.value()).column); + ASSERT_EQ(positions.size(), 1); + EXPECT_EQ(positions.get_element(0), 0); +} + TEST_F(ParquetScanTest, AggregateMinMaxRejectsInexactBinaryStatistics) { write_binary_minmax_parquet_file(_file_path); auto reader = create_reader(); @@ -943,6 +1325,37 @@ TEST_F(ParquetScanTest, GlobalRowIdUsesFileLocalPositionForScanRange) { EXPECT_EQ(row_ids, std::vector({2, 3})); } +TEST_F(ParquetScanTest, PredicateOnlyGlobalRowIdKeepsSignedFileLocalId) { + write_int_pair_parquet_file(_file_path, 6, false); + format::GlobalRowIdContext context {.version = 7, .backend_id = 123456789, .file_id = 42}; + auto reader = create_reader(0, -1, nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + const auto global_rowid = format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(global_rowid).ok()); + request->predicate_only_columns.push_back(global_rowid); + const auto global_rowid_position = request->local_positions.at(global_rowid); + request->conjuncts.push_back(create_always_true_single_column_conjunct( + cast_set(global_rowid_position.value()))); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(rows, 6); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {1, 2, 3, 4, 5, 6})); +} + TEST_F(ParquetScanTest, EmptyScanPlanReturnsEofWithoutReadingColumns) { write_int_pair_parquet_file(_file_path, 2); auto reader = create_reader(); @@ -1008,7 +1421,8 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { while (!eof) { Block block = build_file_block(schema); size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } @@ -1027,6 +1441,9 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); EXPECT_EQ(counter_value(profile, "SelectedRows"), 2); EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 4); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 1); + EXPECT_GT(counter_value(profile, "PredicateCompactionBytes"), 0); + ASSERT_NE(profile.get_counter("PredicateCompactionTime"), nullptr); EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 10); EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 4); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 2); @@ -1068,7 +1485,7 @@ TEST_F(ParquetScanTest, PredicateColumnsSkipUnreadColumnsWhenFirstPredicateFilte EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 6); } -TEST_F(ParquetScanTest, MultiColumnPredicateWaitsForAllPredicateColumns) { +TEST_F(ParquetScanTest, PredicateOnlyColumnDropsPayloadAfterFiltering) { write_int_pair_parquet_file(_file_path, 6, false); RuntimeProfile profile("profile"); auto reader = create_reader(0, -1, &profile); @@ -1080,41 +1497,25 @@ TEST_F(ParquetScanTest, MultiColumnPredicateWaitsForAllPredicateColumns) { auto request = std::make_shared(); format::FileScanRequestBuilder request_builder(request.get()); ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); - request->conjuncts.push_back(create_int32_pair_sum_conjunct(0, 1, 45)); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 2)); ASSERT_TRUE(reader->open(request).ok()); - std::vector ids; - std::vector scores; + Block block = build_file_block(schema); + size_t rows = 0; bool eof = false; - while (!eof) { - Block block = build_file_block(schema); - size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); - if (rows == 0) { - continue; - } - const auto& id_column = int32_data_column(*block.get_by_position(0).column); - const auto& score_column = int32_data_column(*block.get_by_position(1).column); - ASSERT_EQ(id_column.size(), rows); - ASSERT_EQ(score_column.size(), rows); - for (size_t row = 0; row < rows; ++row) { - ids.push_back(id_column.get_element(row)); - scores.push_back(score_column.get_element(row)); - } - } - - EXPECT_EQ(ids, std::vector({1, 2, 3, 4})); - EXPECT_EQ(scores, std::vector({10, 20, 30, 40})); - EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); - EXPECT_EQ(counter_value(profile, "SelectedRows"), 4); - EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 2); - EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 12); - EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 0); + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); } -TEST_F(ParquetScanTest, ProfileCountersReflectPageIndexAndRangeGapPruning) { - write_page_index_parquet_file(_file_path); +TEST_F(ParquetScanTest, PredicateOnlyPlainComparisonUsesPhysicalDirectPath) { + write_int_pair_parquet_file(_file_path, 6, false); RuntimeProfile profile("profile"); auto reader = create_reader(0, -1, &profile); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -1125,11 +1526,511 @@ TEST_F(ParquetScanTest, ProfileCountersReflectPageIndexAndRangeGapPruning) { auto request = std::make_shared(); format::FileScanRequestBuilder request_builder(request.get()); ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 2)); ASSERT_TRUE(reader->open(request).ok()); - size_t total_rows = 0; - bool eof = false; + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, ProjectedPlainComparisonUsesPhysicalFilterAndProjectPath) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {3, 4, 5, 6})); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, ProjectedByteStreamSplitUsesFixedWidthFilterAndProjectPath) { + write_int_pair_parquet_file(_file_path, 6, false, ::parquet::Encoding::BYTE_STREAM_SPLIT); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {3, 4, 5, 6})); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, ProjectedDeltaBinaryPackedUsesFixedWidthFilterAndProjectPath) { + write_int_pair_parquet_file(_file_path, 6, false, ::parquet::Encoding::DELTA_BINARY_PACKED); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {3, 4, 5, 6})); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + +TEST_F(ParquetScanTest, PlainPredicateDirectPathCrossesScratchProbeCadence) { + constexpr int64_t ROWS = 32; + std::vector ids(ROWS); + std::vector scores(ROWS); + std::iota(ids.begin(), ids.end(), 1); + std::iota(scores.begin(), scores.end(), 10); + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("score", arrow::int32(), false), + }); + auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(scores)}); + write_table(_file_path, table, ROWS, false, false, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector file_schema; + ASSERT_TRUE(reader->get_schema(&file_schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(file_schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, ROWS); + // Direct predicate evaluation must survive multiple 16-batch scratch probes; otherwise this + // path can retain a previous outlier for the whole row group without ever aging its capacity. + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), ROWS); +} + +TEST_F(ParquetScanTest, PredicateOnlyUint32FallsBackBeforeRawPlainDecode) { + write_uint32_pair_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_BIGINT); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back( + create_int64_direct_greater_conjunct(0, std::numeric_limits::max())); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows, 2); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {20, 30})); + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 0); +} + +TEST_F(ParquetScanTest, FixedWidthPredicateReportsUnsupportedEncodingAfterProgress) { + write_misdeclared_two_page_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("encoding"), std::string::npos) << status; +} + +TEST_F(ParquetScanTest, PlainDirectPathKeepsPayloadForMultiColumnResidual) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + request->conjuncts.push_back(create_int32_pair_sum_conjunct(0, 1, 42)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + // A residual expression still consumes id, so raw filtering must leave its payload available. + EXPECT_EQ(counter_value(profile, "FixedWidthPredicateDirectBatches"), 0); +} + +// Scenario: every physical batch in every row group is rejected. Predicate readers reach each row +// group boundary, while the lazy score reader remains at row 0. The boundary reset must discard that +// reader and its pending lag instead of issuing SkipRecords for values that can never be observed. +TEST_F(ParquetScanTest, FullyFilteredRowGroupsDropPendingLazyReaders) { + write_int_pair_parquet_file(_file_path, 2, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 100)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + + EXPECT_EQ(total_rows, 0); + // The first one-row probe grows after rejection and remains grown across consecutive empty + // row groups, so later two-row groups are consumed in one predicate batch each. + EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 4); + EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); +} + +// Scenario: row group 0 is fully filtered and leaves two pending lazy rows. Reset must discard that +// lag before row group 1 creates fresh readers at its own row 0; otherwise score 30 would be skipped +// as if it belonged to the rejected prefix from the previous row group. +TEST_F(ParquetScanTest, PendingLazySkipDoesNotCrossRowGroupReset) { + write_int_pair_parquet_file(_file_path, 2, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector scores; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& score_column = int32_data_column(*block.get_by_position(1).column); + for (size_t row = 0; row < rows; ++row) { + scores.push_back(score_column.get_element(row)); + } + } + + EXPECT_EQ(scores, std::vector({30, 40, 50, 60})); + EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); +} + +TEST_F(ParquetScanTest, LongFilteredPrefixSkipsMultipleLazyColumnsInBoundedChunks) { + constexpr size_t ROWS = 70000; + write_long_prefix_parquet_file(_file_path, ROWS); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(32); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(2)).ok()); + request->conjuncts.push_back( + create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, ROWS - 1)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_element(0), ROWS - 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_element(0), ROWS + 9); + EXPECT_EQ(int32_data_column(*block.get_by_position(2).column).get_element(0), ROWS + 19); + EXPECT_LT(counter_value(profile, "TotalBatches"), 32); +} + +TEST_F(ParquetScanTest, EmptyPrefixNeverWidensReturnedBatchPastCallerCap) { + constexpr size_t ROWS = 300; + write_long_prefix_parquet_file(_file_path, ROWS); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(32); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(2)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + int32_t expected_id = 32; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + // The first 32-row probe is empty. Its feedback may accelerate later predicate work, but + // each pending-output slice must preserve the caller's memory-derived row cap. + ASSERT_LE(rows, 32); + const auto& ids = int32_data_column(*block.get_by_position(0).column); + const auto& first_lazy = int32_data_column(*block.get_by_position(1).column); + const auto& second_lazy = int32_data_column(*block.get_by_position(2).column); + ASSERT_EQ(first_lazy.size(), rows); + ASSERT_EQ(second_lazy.size(), rows); + for (size_t row = 0; row < rows; ++row, ++expected_id) { + EXPECT_EQ(ids.get_element(row), expected_id); + EXPECT_EQ(first_lazy.get_element(row), expected_id + 10); + EXPECT_EQ(second_lazy.get_element(row), expected_id + 20); + } + total_rows += rows; + } + EXPECT_EQ(total_rows, ROWS - 32); + EXPECT_EQ(expected_id, ROWS); +} + +// Scenario: a nested lazy column stays behind while id=1 is rejected. Flushing skip(1) must consume +// the complete repetition/definition-level span for the first list, then materialize the remaining +// two parent rows without corrupting their child boundaries. +TEST_F(ParquetScanTest, PendingLazySkipPreservesNestedRowBoundaries) { + write_int_list_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(1); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 1)); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector list_lengths; + std::vector list_values; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + const IColumn* list_column = block.get_by_position(1).column.get(); + if (const auto* nullable = check_and_get_column(list_column)) { + list_column = &nullable->get_nested_column(); + } + const auto& array_column = assert_cast(*list_column); + const auto& value_column = int32_data_column(array_column.get_data()); + for (size_t row = 0; row < rows; ++row) { + const size_t list_start = array_column.offset_at(row); + const size_t list_length = array_column.size_at(row); + list_lengths.push_back(list_length); + for (size_t element = 0; element < list_length; ++element) { + list_values.push_back(value_column.get_element(list_start + element)); + } + } + } + + EXPECT_EQ(list_lengths, std::vector({1, 0})); + EXPECT_EQ(list_values, std::vector({3})); + EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 1); +} + +TEST_F(ParquetScanTest, MultiColumnPredicateWaitsForAllPredicateColumns) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + request->conjuncts.push_back(create_int32_pair_sum_conjunct(0, 1, 45)); + ASSERT_TRUE(reader->open(request).ok()); + + std::vector ids; + std::vector scores; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + const auto& id_column = int32_data_column(*block.get_by_position(0).column); + const auto& score_column = int32_data_column(*block.get_by_position(1).column); + ASSERT_EQ(id_column.size(), rows); + ASSERT_EQ(score_column.size(), rows); + for (size_t row = 0; row < rows; ++row) { + ids.push_back(id_column.get_element(row)); + scores.push_back(score_column.get_element(row)); + } + } + + EXPECT_EQ(ids, std::vector({1, 2, 3, 4})); + EXPECT_EQ(scores, std::vector({10, 20, 30, 40})); + EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); + EXPECT_EQ(counter_value(profile, "SelectedRows"), 4); + EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 12); + EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 0); +} + +TEST_F(ParquetScanTest, ProfileCountersReflectPageIndexAndRangeGapPruning) { + write_page_index_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; while (!eof) { Block block = build_file_block(schema); size_t rows = 0; @@ -1154,5 +2055,30 @@ TEST_F(ParquetScanTest, ProfileCountersReflectPageIndexAndRangeGapPruning) { EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); } +TEST_F(ParquetScanTest, OpenDefersPageIndexProbeToCurrentRowGroup) { + write_multi_row_group_page_index_parquet_file(_file_path); + RuntimeProfile profile("lazy_page_index_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, -1)); + ASSERT_TRUE(reader->open(request).ok()); + + // LIMIT can stop after the first block, so open must not pay remote index I/O for later groups. + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 128); + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 1); +} + } // namespace } // namespace doris diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index e620ed718efbf2..74781c0e532b31 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -16,512 +16,706 @@ // under the License. #include -#include +#include #include +#include #include #include "core/assert_cast.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" namespace doris::format::parquet { -namespace { - -std::vector> build_fields( - const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - std::vector> fields; - EXPECT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - return fields; +TEST(ParquetSchemaTest, NativeMetadataAcceptsRequiredRootWithoutColumns) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(0); + root.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root}).ok()); + EXPECT_EQ(descriptor.size(), 0); + + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root}); + thrift_metadata.__set_num_rows(0); + NativeParquetMetadata metadata(std::move(thrift_metadata), 0); + // An empty physical tree remains useful for metadata-only COUNT(*); rejecting it here changes + // the compatibility contract before request planning can select that path. + EXPECT_TRUE(metadata.init_schema(false, false).ok()); + EXPECT_EQ(metadata.schema().size(), 0); } -Status build_status(const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - std::vector> fields; - return build_parquet_column_schema(descriptor, &fields); -} +TEST(ParquetSchemaTest, NativeMetadataTreePreservesNestedFieldNamesAndIds) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); -} // namespace - -TEST(ParquetSchemaTest, PrimitiveStateAndFieldIdArePreserved) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("required_i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("optional_i64", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT64, - ::parquet::ConvertedType::NONE, -1, -1, -1, 42), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->local_id, 0); - EXPECT_EQ(fields[0]->name, "required_i32"); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->leaf_column_id, 0); - EXPECT_EQ(fields[0]->nullable_definition_level, 0); - EXPECT_FALSE(fields[0]->type->is_nullable()); - - EXPECT_EQ(fields[1]->local_id, 1); - EXPECT_EQ(fields[1]->parquet_field_id, 42); - EXPECT_EQ(fields[1]->leaf_column_id, 1); - EXPECT_EQ(fields[1]->nullable_definition_level, 1); - EXPECT_TRUE(fields[1]->type->is_nullable()); -} + tparquet::SchemaElement protocol; + protocol.__set_name("protocol"); + protocol.__set_num_children(2); + protocol.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + protocol.__set_field_id(10); -TEST(ParquetSchemaTest, PrimitiveTypeDescriptorCoversLogicalConvertedAndPhysicalFallback) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(false, - ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64), - ::parquet::schema::PrimitiveNode::Make("i8", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8), - ::parquet::schema::PrimitiveNode::Make("plain", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - }); - - ASSERT_EQ(fields.size(), 3); - EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(fields[0]->type_descriptor.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - EXPECT_TRUE(fields[0]->type_descriptor.is_timestamp); - EXPECT_FALSE(fields[0]->type_descriptor.timestamp_is_adjusted_to_utc); - - EXPECT_EQ(remove_nullable(fields[1]->type)->get_primitive_type(), TYPE_TINYINT); - EXPECT_EQ(fields[1]->type_descriptor.integer_bit_width, 8); - EXPECT_FALSE(fields[1]->type_descriptor.is_unsigned_integer); - - EXPECT_EQ(remove_nullable(fields[2]->type)->get_primitive_type(), TYPE_DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.physical_type, ::parquet::Type::DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::NONE); -} + tparquet::SchemaElement min_reader; + min_reader.__set_name("minReaderVersion"); + min_reader.__set_type(tparquet::Type::INT32); + min_reader.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + min_reader.__set_field_id(11); + + tparquet::SchemaElement min_writer = min_reader; + min_writer.__set_name("minWriterVersion"); + min_writer.__set_field_id(12); -TEST(ParquetSchemaTest, StructMakesDataTypeChildrenNullableAndPropagatesLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::PrimitiveNode::Make("a", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - })}); + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, protocol, min_reader, min_writer}).ok()); + native_schema.assign_ids(); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); ASSERT_EQ(fields.size(), 1); - const auto& struct_schema = *fields[0]; - EXPECT_EQ(struct_schema.kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(struct_schema.nullable_definition_level, 1); - ASSERT_EQ(struct_schema.children.size(), 2); - EXPECT_EQ(struct_schema.children[0]->definition_level, 1); - EXPECT_EQ(struct_schema.children[1]->definition_level, 2); - EXPECT_EQ(struct_schema.max_definition_level, 2); - - const auto& struct_type = - assert_cast(*remove_nullable(struct_schema.type)); - ASSERT_EQ(struct_type.get_elements().size(), 2); - EXPECT_TRUE(struct_type.get_elements()[0]->is_nullable()); - EXPECT_TRUE(struct_type.get_elements()[1]->is_nullable()); + EXPECT_EQ(fields[0]->name, "protocol"); + EXPECT_EQ(fields[0]->parquet_field_id, 10); + ASSERT_EQ(fields[0]->children.size(), 2); + EXPECT_EQ(fields[0]->children[0]->name, "minReaderVersion"); + EXPECT_EQ(fields[0]->children[0]->leaf_column_id, 0); + EXPECT_EQ(fields[0]->children[1]->name, "minWriterVersion"); + EXPECT_EQ(fields[0]->children[1]->leaf_column_id, 1); + + std::shared_ptr mapping; + ASSERT_TRUE(build_native_schema_node(fields[0]->type, *fields[0], &mapping).ok()); + EXPECT_TRUE(mapping->has_child("minReaderVersion")); + EXPECT_EQ(mapping->file_child_name("minReaderVersion"), "minReaderVersion"); + ASSERT_NE(mapping->child("minReaderVersion"), nullptr); } - -TEST(ParquetSchemaTest, ListCompatibilityRulesAndLevels) { - const auto standard_list = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - const auto structural_array = ::parquet::schema::GroupNode::Make( - "ys", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "array", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({standard_list, structural_array}); - ASSERT_EQ(fields.size(), 2); - - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(xs.definition_level, 2); - EXPECT_EQ(xs.repetition_level, 1); - ASSERT_EQ(xs.children.size(), 1); - EXPECT_EQ(xs.children[0]->name, "element"); - EXPECT_EQ(xs.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_TRUE(xs.children[0]->type->is_nullable()); - const auto& xs_type = assert_cast(*remove_nullable(xs.type)); - EXPECT_TRUE(xs_type.get_nested_type()->is_nullable()); - - const auto& ys = *fields[1]; - EXPECT_EQ(ys.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(ys.children.size(), 1); - EXPECT_EQ(ys.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(remove_nullable(ys.children[0]->type)->get_primitive_type(), TYPE_STRUCT); +TEST(ParquetSchemaTest, NativeLogicalUtcTimeIsDeferredToProjectionValidation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement adjusted_time; + adjusted_time.__set_name("time_ms"); + adjusted_time.__set_type(tparquet::Type::INT32); + adjusted_time.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + adjusted_time.__set_logicalType(tparquet::LogicalType()); + adjusted_time.logicalType.__set_TIME(tparquet::TimeType()); + adjusted_time.logicalType.TIME.__set_isAdjustedToUTC(true); + adjusted_time.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + adjusted_time.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, adjusted_time}).ok()); + native_schema.assign_ids(); + std::vector> fields; + const auto status = build_parquet_column_schema(native_schema, &fields); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(fields.size(), 1); + // Preserve the unsupported marker in metadata; request-level validation decides whether the + // leaf is a real projection or an ignorable COUNT(*) placeholder. + EXPECT_EQ(fields[0]->type_descriptor.unsupported_reason, + "Parquet TIME with isAdjustedToUTC=true is not supported"); } -TEST(ParquetSchemaTest, LegacyListElementResolutionRulesArePreserved) { - const auto two_level_list = ::parquet::schema::GroupNode::Make( - "two_level", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - const auto tuple_list = ::parquet::schema::GroupNode::Make( - "tuple_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "tuple_list_tuple", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - const auto multi_field_list = ::parquet::schema::GroupNode::Make( - "records", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("name", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::LIST); - const auto fields = build_fields({two_level_list, tuple_list, multi_field_list}); - ASSERT_EQ(fields.size(), 3); - - const auto& two_level = *fields[0]; - EXPECT_EQ(two_level.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(two_level.definition_level, 2); - EXPECT_EQ(two_level.repetition_level, 1); - ASSERT_EQ(two_level.children.size(), 1); - EXPECT_EQ(two_level.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(two_level.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(two_level.children[0]->type)->get_primitive_type(), TYPE_INT); - - const auto& tuple = *fields[1]; - ASSERT_EQ(tuple.children.size(), 1); - EXPECT_EQ(tuple.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(tuple.children[0]->name, "element"); - ASSERT_EQ(tuple.children[0]->children.size(), 1); - EXPECT_EQ(tuple.children[0]->children[0]->name, "value"); - - const auto& multi_field = *fields[2]; - ASSERT_EQ(multi_field.children.size(), 1); - EXPECT_EQ(multi_field.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(multi_field.children[0]->children.size(), 2); - EXPECT_EQ(multi_field.children[0]->children[0]->name, "id"); - EXPECT_EQ(multi_field.children[0]->children[1]->name, "name"); +TEST(ParquetSchemaTest, NativeOversizedByteArrayDecimalIsDeferredToProjectionValidation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement decimal; + decimal.__set_name("decimal_77"); + decimal.__set_type(tparquet::Type::BYTE_ARRAY); + decimal.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + decimal.__set_logicalType(tparquet::LogicalType()); + decimal.logicalType.__set_DECIMAL(tparquet::DecimalType()); + decimal.logicalType.DECIMAL.__set_precision(77); + decimal.logicalType.DECIMAL.__set_scale(2); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, decimal}).ok()); + native_schema.assign_ids(); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); + ASSERT_EQ(fields.size(), 1); + EXPECT_TRUE(fields[0]->type_descriptor.is_decimal); + EXPECT_FALSE(fields[0]->type_descriptor.unsupported_reason.empty()); + EXPECT_NE(fields[0]->type_descriptor.unsupported_reason.find("precision 77"), + std::string::npos); } -TEST(ParquetSchemaTest, NestedRepeatedInsideListElementIsWrappedOnce) { - const auto list_with_repeated_child = ::parquet::schema::GroupNode::Make( - "outer", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({list_with_repeated_child}); +TEST(ParquetSchemaTest, NativeUnknownLogicalTypeRetainsPhysicalFallback) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement interval; + interval.__set_name("duration"); + interval.__set_type(tparquet::Type::FIXED_LEN_BYTE_ARRAY); + interval.__set_type_length(12); + interval.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + interval.__set_logicalType(tparquet::LogicalType()); + interval.__set_converted_type(tparquet::ConvertedType::INTERVAL); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, interval}).ok()); + native_schema.assign_ids(); + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); ASSERT_EQ(fields.size(), 1); - const auto& outer = *fields[0]; - EXPECT_EQ(outer.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(outer.children.size(), 1); - const auto& element = *outer.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.children[0]->name, "items"); - ASSERT_EQ(element.children[0]->children.size(), 1); - EXPECT_EQ(element.children[0]->children[0]->name, "element"); + EXPECT_TRUE(fields[0]->type_descriptor.unsupported_reason.empty()); + EXPECT_EQ(fields[0]->type_descriptor.doris_type->get_primitive_type(), TYPE_STRING); } -TEST(ParquetSchemaTest, ListWrapperWithLogicalAnnotationIsPreservedAsElement) { - const auto annotated_repeated_group = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - EXPECT_FALSE(build_status({annotated_repeated_group}).ok()); - - const auto nested_list_wrapper = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({nested_list_wrapper}); - ASSERT_EQ(fields.size(), 1); - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(xs.children.size(), 1); - const auto& element = *xs.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.name, "element"); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(element.children[0]->type)->get_primitive_type(), TYPE_INT); +TEST(ParquetSchemaTest, NativeGroupPrimitiveLogicalTypesAreRejected) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement child; + child.__set_name("value"); + child.__set_type(tparquet::Type::BYTE_ARRAY); + child.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + std::vector> logical_types; + tparquet::LogicalType string_type; + string_type.__set_STRING(tparquet::StringType()); + logical_types.emplace_back("STRING", string_type); + tparquet::LogicalType enum_type; + enum_type.__set_ENUM(tparquet::EnumType()); + logical_types.emplace_back("ENUM", enum_type); + tparquet::LogicalType decimal_type; + decimal_type.__set_DECIMAL(tparquet::DecimalType()); + logical_types.emplace_back("DECIMAL", decimal_type); + tparquet::LogicalType date_type; + date_type.__set_DATE(tparquet::DateType()); + logical_types.emplace_back("DATE", date_type); + tparquet::LogicalType time_type; + time_type.__set_TIME(tparquet::TimeType()); + logical_types.emplace_back("TIME", time_type); + tparquet::LogicalType timestamp_type; + timestamp_type.__set_TIMESTAMP(tparquet::TimestampType()); + logical_types.emplace_back("TIMESTAMP", timestamp_type); + tparquet::LogicalType integer_type; + integer_type.__set_INTEGER(tparquet::IntType()); + logical_types.emplace_back("INTEGER", integer_type); + tparquet::LogicalType uuid_type; + uuid_type.__set_UUID(tparquet::UUIDType()); + logical_types.emplace_back("UUID", uuid_type); + tparquet::LogicalType float16_type; + float16_type.__set_FLOAT16(tparquet::Float16Type()); + logical_types.emplace_back("FLOAT16", float16_type); + + for (const auto& [name, logical_type] : logical_types) { + SCOPED_TRACE(name); + tparquet::SchemaElement group; + group.__set_name("bad_" + name + "_group"); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + group.__set_logicalType(logical_type); + NativeFieldDescriptor native_schema; + const auto status = native_schema.parse_from_thrift({root, group, child}); + EXPECT_FALSE(status.ok()); + if (name == "ENUM") { + EXPECT_NE(status.to_string().find("Logical type Enum cannot be applied to group node"), + std::string::npos); + } + } + + for (const auto converted_type : + {tparquet::ConvertedType::UTF8, tparquet::ConvertedType::ENUM, + tparquet::ConvertedType::DECIMAL, tparquet::ConvertedType::DATE, + tparquet::ConvertedType::TIME_MILLIS, tparquet::ConvertedType::TIMESTAMP_MICROS, + tparquet::ConvertedType::INT_32, tparquet::ConvertedType::JSON, + tparquet::ConvertedType::BSON}) { + SCOPED_TRACE(tparquet::to_string(converted_type)); + tparquet::SchemaElement group; + group.__set_name("bad_converted_group"); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + group.__set_converted_type(converted_type); + NativeFieldDescriptor native_schema; + EXPECT_FALSE(native_schema.parse_from_thrift({root, group, child}).ok()); + } } -TEST(ParquetSchemaTest, MapWrapperIsFoldedAndOptionalKeyIsAllowed) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); +TEST(ParquetSchemaTest, NativeFlatLeafValueCountMustMatchRowCount) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + tparquet::Statistics statistics; + statistics.__set_null_count(2); + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_num_values(2); + column_metadata.__set_statistics(statistics); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + row_group.__set_columns({chunk}); + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root, leaf}); + thrift_metadata.__set_num_rows(1); + thrift_metadata.__set_row_groups({row_group}); + + NativeParquetMetadata metadata(std::move(thrift_metadata), 0); + const auto status = metadata.init_schema(false, false); + EXPECT_TRUE(status.is()) << status; +} - ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_EQ(map_schema.kind, ParquetColumnSchemaKind::MAP); - EXPECT_EQ(map_schema.definition_level, 2); - EXPECT_EQ(map_schema.repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->name, "key"); - EXPECT_EQ(map_schema.children[1]->name, "value"); - EXPECT_TRUE(map_schema.children[0]->type->is_nullable()); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); +TEST(ParquetSchemaTest, NativeStringAnnotationsAndTimeUnitsPreserveLogicalTypes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(8); + + auto binary_leaf = [](const std::string& name) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return leaf; + }; + auto logical_enum = binary_leaf("logical_enum"); + logical_enum.__set_logicalType(tparquet::LogicalType()); + logical_enum.logicalType.__set_ENUM(tparquet::EnumType()); + auto logical_bson = binary_leaf("logical_bson"); + logical_bson.__set_logicalType(tparquet::LogicalType()); + logical_bson.logicalType.__set_BSON(tparquet::BsonType()); + auto converted_enum = binary_leaf("converted_enum"); + converted_enum.__set_converted_type(tparquet::ConvertedType::ENUM); + auto converted_bson = binary_leaf("converted_bson"); + converted_bson.__set_converted_type(tparquet::ConvertedType::BSON); + + auto logical_time = [](const std::string& name, bool millis) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(millis ? tparquet::Type::INT32 : tparquet::Type::INT64); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + leaf.__set_logicalType(tparquet::LogicalType()); + leaf.logicalType.__set_TIME(tparquet::TimeType()); + leaf.logicalType.TIME.__set_isAdjustedToUTC(false); + leaf.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + if (millis) { + leaf.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + } else { + leaf.logicalType.TIME.unit.__set_MICROS(tparquet::MicroSeconds()); + } + return leaf; + }; + auto logical_millis = logical_time("logical_millis", true); + auto logical_micros = logical_time("logical_micros", false); + auto converted_millis = logical_time("converted_millis", true); + converted_millis.__isset.logicalType = false; + converted_millis.__set_converted_type(tparquet::ConvertedType::TIME_MILLIS); + auto converted_micros = logical_time("converted_micros", false); + converted_micros.__isset.logicalType = false; + converted_micros.__set_converted_type(tparquet::ConvertedType::TIME_MICROS); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor + .parse_from_thrift({root, logical_enum, logical_bson, converted_enum, + converted_bson, logical_millis, logical_micros, + converted_millis, converted_micros}) + .ok()); + for (size_t field = 0; field < 4; ++field) { + EXPECT_EQ(remove_nullable(descriptor.get_column(field)->data_type)->get_primitive_type(), + TYPE_STRING); + } + EXPECT_EQ(remove_nullable(descriptor.get_column(4)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(5)->data_type)->get_scale(), 6); + EXPECT_EQ(remove_nullable(descriptor.get_column(6)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(7)->data_type)->get_scale(), 6); } -TEST(ParquetSchemaTest, StandardMapLevelsAndDataTypesAreBuiltFromEntryContext) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::REQUIRED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); +TEST(ParquetSchemaTest, NativeSchemaRejectsAmbiguousKindsAndMissingRepetition) { + auto valid_root = []() { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + return root; + }; + auto valid_leaf = []() { + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + return leaf; + }; + + NativeFieldDescriptor descriptor; + auto primitive_root = valid_root(); + primitive_root.__set_type(tparquet::Type::INT32); + EXPECT_FALSE(descriptor.parse_from_thrift({primitive_root, valid_leaf()}).ok()); + + auto repeated_root = valid_root(); + repeated_root.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + EXPECT_FALSE(descriptor.parse_from_thrift({repeated_root, valid_leaf()}).ok()); + + auto missing_repetition = valid_leaf(); + missing_repetition.__isset.repetition_type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_repetition}).ok()); + + auto dual_kind = valid_leaf(); + dual_kind.__set_num_children(1); + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), dual_kind}).ok()); + + auto legacy_zero_children = valid_leaf(); + legacy_zero_children.__set_num_children(0); + EXPECT_TRUE(descriptor.parse_from_thrift({valid_root(), legacy_zero_children}).ok()); + + auto missing_kind = valid_leaf(); + missing_kind.__isset.type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_kind}).ok()); +} - ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_FALSE(map_schema.type->is_nullable()); - EXPECT_EQ(map_schema.definition_level, 1); - EXPECT_EQ(map_schema.repetition_level, 1); - EXPECT_EQ(map_schema.repeated_repetition_level, 1); - EXPECT_EQ(map_schema.max_definition_level, 2); - EXPECT_EQ(map_schema.max_repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->definition_level, 1); - EXPECT_EQ(map_schema.children[0]->repetition_level, 1); - EXPECT_EQ(map_schema.children[1]->definition_level, 2); - EXPECT_EQ(map_schema.children[1]->nullable_definition_level, 2); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); +TEST(ParquetSchemaTest, NativeSchemaRejectsUnboundedChildCountsBeforeAllocation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(std::numeric_limits::max()); + NativeFieldDescriptor descriptor; + EXPECT_FALSE(descriptor.parse_from_thrift({root}).ok()); + + root.__set_num_children(1); + tparquet::SchemaElement nested; + nested.__set_name("nested"); + nested.__set_num_children(std::numeric_limits::max()); + nested.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + EXPECT_FALSE(descriptor.parse_from_thrift({root, nested}).ok()); } -TEST(ParquetSchemaTest, BareRepeatedFieldsAreWrappedAsLists) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("items", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32), - ::parquet::schema::GroupNode::Make( - "links", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("url", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("rank", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[0]->children.size(), 1); - EXPECT_EQ(fields[0]->children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->children[0]->name, "element"); - - EXPECT_EQ(fields[1]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[1]->children.size(), 1); - EXPECT_EQ(fields[1]->children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(fields[1]->children[0]->name, "element"); +std::vector nested_native_schema(size_t depth) { + std::vector schema; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + schema.push_back(root); + for (size_t level = 0; level < depth; ++level) { + tparquet::SchemaElement group; + group.__set_name("g" + std::to_string(level)); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(group); + } + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(leaf); + return schema; } -TEST(ParquetSchemaTest, DeepLevelChainPropagatesDefinitionAndRepetitionLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "inner", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})})}); +TEST(ParquetSchemaTest, NativeSchemaBoundsRecursiveDepth) { + NativeFieldDescriptor accepted; + EXPECT_TRUE(accepted.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH)).ok()); + NativeFieldDescriptor rejected; + EXPECT_FALSE( + rejected.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH + 1)).ok()); +} - ASSERT_EQ(fields.size(), 1); - const auto& s = *fields[0]; - EXPECT_EQ(s.definition_level, 1); - EXPECT_EQ(s.nullable_definition_level, 1); - ASSERT_EQ(s.children.size(), 1); - const auto& inner = *s.children[0]; - EXPECT_EQ(inner.definition_level, 2); - EXPECT_EQ(inner.nullable_definition_level, 2); - ASSERT_EQ(inner.children.size(), 1); - const auto& items = *inner.children[0]; - EXPECT_EQ(items.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(items.definition_level, 3); - EXPECT_EQ(items.repetition_level, 1); - EXPECT_EQ(items.repeated_ancestor_definition_level, 3); - EXPECT_EQ(items.repeated_repetition_level, 1); - EXPECT_EQ(items.max_definition_level, 3); - EXPECT_EQ(items.max_repetition_level, 1); - ASSERT_EQ(items.children.size(), 1); - EXPECT_EQ(items.children[0]->definition_level, 3); - EXPECT_EQ(items.children[0]->repetition_level, 1); +TEST(ParquetSchemaTest, NativeListTupleCompatibilityRequiresEnclosingListName) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto list = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + schema.__set_converted_type(tparquet::ConvertedType::LIST); + return schema; + }; + auto wrapper = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + return schema; + }; + auto item = []() { + tparquet::SchemaElement schema; + schema.__set_name("item"); + schema.__set_type(tparquet::Type::INT32); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return schema; + }; + + NativeFieldDescriptor mismatched; + ASSERT_TRUE(mismatched.parse_from_thrift({root(), list("xs"), wrapper("other_tuple"), item()}) + .ok()); + const auto* mismatched_element = &mismatched.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(mismatched_element->data_type)->get_primitive_type(), TYPE_INT); + + NativeFieldDescriptor matching; + ASSERT_TRUE(matching.parse_from_thrift({root(), list("xs"), wrapper("xs_tuple"), item()}).ok()); + const auto* matching_element = &matching.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(matching_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(matching_element->children.size(), 1); + EXPECT_EQ(matching_element->children[0].name, "item"); } -TEST(ParquetSchemaTest, BuildEntryValidatesNullPointerAndEmptyRoot) { - auto empty_root = ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, - ::parquet::schema::NodeVector {}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(empty_root); +TEST(ParquetSchemaTest, NativeListPreservesRepeatedAndAnnotatedElementWrappers) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto group = [](const std::string& name, tparquet::FieldRepetitionType::type repetition) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(repetition); + return schema; + }; + auto outer_list = group("outer", tparquet::FieldRepetitionType::OPTIONAL); + outer_list.__set_converted_type(tparquet::ConvertedType::LIST); + + auto repeated_wrapper = group("list", tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement repeated_items; + repeated_items.__set_name("items"); + repeated_items.__set_type(tparquet::Type::INT32); + repeated_items.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + NativeFieldDescriptor repeated; + ASSERT_TRUE(repeated.parse_from_thrift({root(), outer_list, repeated_wrapper, repeated_items}) + .ok()); + const auto* repeated_element = &repeated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(repeated_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(repeated_element->children.size(), 1); + EXPECT_EQ(repeated_element->children[0].name, "items"); + EXPECT_EQ(remove_nullable(repeated_element->children[0].data_type)->get_primitive_type(), + TYPE_ARRAY); + + auto annotated_wrapper = repeated_wrapper; + annotated_wrapper.__set_converted_type(tparquet::ConvertedType::LIST); + auto nested_wrapper = repeated_wrapper; + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::INT32); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + NativeFieldDescriptor annotated; + ASSERT_TRUE(annotated + .parse_from_thrift( + {root(), outer_list, annotated_wrapper, nested_wrapper, value}) + .ok()); + const auto* annotated_element = &annotated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(annotated_element->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(annotated_element->children.size(), 1); + EXPECT_EQ(remove_nullable(annotated_element->children[0].data_type)->get_primitive_type(), + TYPE_INT); +} + +TEST(ParquetSchemaTest, NativeSchemaRecognizesLogicalTypeOnlyListAndMap) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(2); + + tparquet::SchemaElement list; + list.__set_name("items"); + list.__set_num_children(1); + list.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + list.__set_logicalType(tparquet::LogicalType()); + list.logicalType.__set_LIST(tparquet::ListType()); + tparquet::SchemaElement list_wrapper; + list_wrapper.__set_name("list"); + list_wrapper.__set_num_children(1); + list_wrapper.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement element; + element.__set_name("element"); + element.__set_type(tparquet::Type::INT32); + element.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + tparquet::SchemaElement map; + map.__set_name("attributes"); + map.__set_num_children(1); + map.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + map.__set_logicalType(tparquet::LogicalType()); + map.logicalType.__set_MAP(tparquet::MapType()); + tparquet::SchemaElement key_value; + key_value.__set_name("key_value"); + key_value.__set_num_children(2); + key_value.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement key; + key.__set_name("key"); + key.__set_type(tparquet::Type::BYTE_ARRAY); + key.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::INT64); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor + .parse_from_thrift( + {root, list, list_wrapper, element, map, key_value, key, value}) + .ok()); + ASSERT_EQ(descriptor.size(), 2); + EXPECT_EQ(remove_nullable(descriptor.get_column(0)->data_type)->get_primitive_type(), + TYPE_ARRAY); + EXPECT_EQ(remove_nullable(descriptor.get_column(1)->data_type)->get_primitive_type(), TYPE_MAP); +} - EXPECT_FALSE(build_parquet_column_schema(descriptor, nullptr).ok()); +TEST(ParquetSchemaTest, NativeMetadataRejectsRowGroupChunkCardinalityAndMissingMetadata) { + auto make_metadata = []() { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(2); + tparquet::SchemaElement first; + first.__set_name("a"); + first.__set_type(tparquet::Type::INT32); + first.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement second = first; + second.__set_name("b"); + metadata.__set_schema({root, first, second}); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + return metadata; + }; + + { + NativeParquetMetadata metadata(make_metadata(), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } + { + auto thrift = make_metadata(); + thrift.row_groups[0].columns.resize(2); + tparquet::ColumnMetaData first_meta; + first_meta.__set_type(tparquet::Type::INT32); + thrift.row_groups[0].columns[0].__set_meta_data(first_meta); + NativeParquetMetadata metadata(std::move(thrift), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } +} +TEST(ParquetSchemaTest, NativeProjectionUsesResolvedIdentityBeforeCaseInsensitiveFallback) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement group; + group.__set_name("s"); + group.__set_num_children(3); + group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + tparquet::SchemaElement upper; + upper.__set_name("Value"); + upper.__set_type(tparquet::Type::INT32); + upper.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + upper.__set_field_id(1); + tparquet::SchemaElement lower = upper; + lower.__set_name("value"); + lower.__set_field_id(2); + tparquet::SchemaElement other = upper; + other.__set_name("other"); + other.__set_field_id(3); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, group, upper, lower, other}).ok()); + descriptor.assign_ids(); std::vector> fields; ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - EXPECT_TRUE(fields.empty()); -} -TEST(ParquetSchemaTest, RejectInvalidListMapAndUnsupportedTime) { - const auto bad_list = ::parquet::schema::GroupNode::Make( - "bad_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({bad_list}).ok()); - - const auto bad_map = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("entry", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({bad_map}).ok()); - - const auto converted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS); - const auto status = build_status({converted_time}); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), - std::string::npos); + const auto int_type = make_nullable(std::make_shared()); + std::shared_ptr mapping; + auto other_only = make_nullable( + std::make_shared(DataTypes {int_type}, Strings {"other"})); + ASSERT_TRUE(build_native_schema_node(other_only, *fields[0], &mapping).ok()); + EXPECT_TRUE(mapping->has_child("other")); + + auto exact_case = make_nullable( + std::make_shared(DataTypes {int_type}, Strings {"Value"})); + ASSERT_TRUE(build_native_schema_node(exact_case, *fields[0], &mapping).ok()); + EXPECT_EQ(mapping->file_child_name("Value"), "Value"); + + auto ambiguous_fallback = make_nullable( + std::make_shared(DataTypes {int_type}, Strings {"VALUE"})); + const auto status = build_native_schema_node(ambiguous_fallback, *fields[0], &mapping); + EXPECT_TRUE(status.is()) << status; } -TEST(ParquetSchemaTest, RejectAdditionalInvalidListAndMapLayouts) { - const auto zero_child_list = ::parquet::schema::GroupNode::Make( - "zero_child_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make("list", ::parquet::Repetition::REPEATED, - ::parquet::schema::NodeVector {})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({zero_child_list}).ok()); - - const auto repeated_list = ::parquet::schema::GroupNode::Make( - "repeated_list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({repeated_list}).ok()); - - const auto map_with_two_fields = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::GroupNode::Make( - "entry1", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - ::parquet::schema::GroupNode::Make( - "entry2", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_with_two_fields}).ok()); - - const auto non_repeated_map_entry = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({non_repeated_map_entry}).ok()); - - const auto map_entry_with_one_child = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_entry_with_one_child}).ok()); - - const auto repeated_map = ::parquet::schema::GroupNode::Make( - "repeated_map", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({repeated_map}).ok()); +TEST(ParquetSchemaTest, NativeSetMapKeyValueWrapperRemainsSingleListElement) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement set; + set.__set_name("tags"); + set.__set_num_children(1); + set.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + set.__set_converted_type(tparquet::ConvertedType::MAP); + tparquet::SchemaElement wrapper; + wrapper.__set_name("key_value"); + wrapper.__set_num_children(1); + wrapper.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + wrapper.__set_converted_type(tparquet::ConvertedType::MAP_KEY_VALUE); + tparquet::SchemaElement key; + key.__set_name("key"); + key.__set_type(tparquet::Type::INT32); + key.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, set, wrapper, key}).ok()); + const auto* column = descriptor.get_column(0); + EXPECT_EQ(remove_nullable(column->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(column->children.size(), 1); + EXPECT_EQ(remove_nullable(column->children[0].data_type)->get_primitive_type(), TYPE_INT); } -TEST(ParquetSchemaTest, LogicalUtcTimeIsRejected) { - const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32); - const auto status = build_status({adjusted_time}); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), - std::string::npos); +TEST(ParquetSchemaTest, NativeListPreservesLegacyMapKeyValueElement) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement list; + list.__set_name("entries"); + list.__set_num_children(1); + list.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + list.__set_converted_type(tparquet::ConvertedType::LIST); + tparquet::SchemaElement element; + element.__set_name("element"); + element.__set_num_children(1); + element.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + element.__set_converted_type(tparquet::ConvertedType::MAP_KEY_VALUE); + tparquet::SchemaElement map; + map.__set_name("map"); + map.__set_num_children(2); + map.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement key; + key.__set_name("key"); + key.__set_type(tparquet::Type::INT32); + key.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::BYTE_ARRAY); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor.parse_from_thrift({root, list, element, map, key, value}).ok()); + const auto* column = descriptor.get_column(0); + EXPECT_EQ(remove_nullable(column->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(column->children.size(), 1); + EXPECT_EQ(remove_nullable(column->children[0].data_type)->get_primitive_type(), TYPE_MAP); + ASSERT_EQ(column->children[0].children.size(), 2); + EXPECT_EQ(remove_nullable(column->children[0].children[0].data_type)->get_primitive_type(), + TYPE_INT); + EXPECT_EQ(remove_nullable(column->children[0].children[1].data_type)->get_primitive_type(), + TYPE_STRING); } } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp b/be/test/format_v2/parquet/parquet_serde_reader_test.cpp deleted file mode 100644 index c35138e3263723..00000000000000 --- a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp +++ /dev/null @@ -1,459 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_decimal.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_nullable.h" -#include "core/types.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -class ParquetSerdeReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_serde_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "serde.parquet").string(); - write_parquet_file(); - open_file(_file_path); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(0x0000).ok()); - EXPECT_TRUE(builder.Append(0x8000).ok()); - EXPECT_TRUE(builder.Append(0x3E00).ok()); - EXPECT_TRUE(builder.Append(0x7E00).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_binary_array(const std::vector& values) { - arrow::BinaryBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data()), - static_cast(value.size())) - .ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_fixed_binary_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_timestamp_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_decimal_array(const std::shared_ptr& type, - const std::vector& values) { - arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); - } - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, - std::shared_ptr array) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - } - - void write_table(const std::string& file_path, const std::shared_ptr& table, - std::shared_ptr<::parquet::ArrowWriterProperties> arrow_properties = nullptr) { - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - ::parquet::WriterProperties::Builder writer_builder; - writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - writer_builder.compression(::parquet::Compression::UNCOMPRESSED); - if (arrow_properties == nullptr) { - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_properties = arrow_builder.build(); - } - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), *file_result, ROW_COUNT, - writer_builder.build(), std::move(arrow_properties))); - } - - void write_parquet_file() { - add_field(arrow::field("bool_col", arrow::boolean(), false), - build_required_array( - {true, false, true, false, true})); - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50})); - add_field(arrow::field("int64_col", arrow::int64(), false), - build_required_array( - {10000000000L, -9L, 42L, 77L, 123L})); - add_field(arrow::field("uint32_col", arrow::uint32(), false), - build_required_array( - {0U, 1U, 1U << 31, std::numeric_limits::max(), 42U})); - add_field(arrow::field("uint64_col", arrow::uint64(), false), - build_required_array( - {0ULL, 1ULL, 1ULL << 63, std::numeric_limits::max(), 42ULL})); - add_field(arrow::field("float_col", arrow::float32(), false), - build_required_array( - {1.5F, -2.25F, 3.0F, 4.5F, 5.75F})); - add_field(arrow::field("double_col", arrow::float64(), false), - build_required_array({3.5, -4.75, 6.0, 7.25, 8.5})); - add_field(arrow::field("nullable_float16_col", arrow::float16(), true), - build_nullable_float16_array()); - add_field(arrow::field("binary_col", arrow::binary(), false), - build_binary_array({"bin_a", "bin_b", "bin_c", "bin_d", "bin_e"})); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"})); - add_field(arrow::field("fixed_binary_col", arrow::fixed_size_binary(4), false), - build_fixed_binary_array(arrow::fixed_size_binary(4), - {"aaaa", "bbbb", "cccc", "dddd", "eeee"})); - add_field(arrow::field("date_col", arrow::date32(), false), - build_required_array({0, 1, 18628, 18629, 18630})); - add_field(arrow::field("timestamp_millis_col", arrow::timestamp(arrow::TimeUnit::MILLI), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MILLI), - {0, 1234, 1609459200000, 1609459201000, -1})); - add_field(arrow::field("timestamp_micros_col", arrow::timestamp(arrow::TimeUnit::MICRO), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("timestamp_micros_utc_col", - arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("decimal_fixed_binary_9_2_col", arrow::decimal128(9, 2), false), - build_decimal_array(arrow::decimal128(9, 2), {12345, -67, 0, 987, 1000})); - add_field(arrow::field("decimal_fixed_binary_18_6_col", arrow::decimal128(18, 6), false), - build_decimal_array(arrow::decimal128(18, 6), - {1234567, -670000, 0, 9870000, 1000000})); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array()); - - write_table(_file_path, arrow::Table::Make(arrow::schema(_arrow_fields), _arrays)); - } - - void open_file(const std::string& file_path) { - _file_reader = ::parquet::ParquetFileReader::OpenFile(file_path, false); - ASSERT_NE(_file_reader, nullptr); - ASSERT_EQ(_file_reader->metadata()->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet serde test field " << name; - return _fields.size(); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - template - void read_and_validate(const std::string& name, Validator validator) const { - const auto field_idx = find_field_idx(name); - ASSERT_TRUE(supports_record_reader(_fields[field_idx]->type_descriptor)); - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - validator(*_fields[field_idx], *column); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; -}; - -TEST_F(ParquetSerdeReaderTest, ReadAllSupportedPhysicalAndLogicalTypes) { - read_and_validate("bool_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BOOLEAN); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 0); - EXPECT_EQ(values.get_element(4), 1); - }); - read_and_validate("int32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - read_and_validate("int64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10000000000L); - EXPECT_EQ(values.get_element(1), -9L); - }); - read_and_validate("uint32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_BIGINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), 2147483648L); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("uint64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_LARGEINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), static_cast(1) << 63); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("float_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FLOAT); - const auto& values = assert_cast(column); - EXPECT_FLOAT_EQ(values.get_element(0), 1.5F); - EXPECT_FLOAT_EQ(values.get_element(1), -2.25F); - }); - read_and_validate("double_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::DOUBLE); - const auto& values = assert_cast(column); - EXPECT_DOUBLE_EQ(values.get_element(0), 3.5); - EXPECT_DOUBLE_EQ(values.get_element(1), -4.75); - }); - read_and_validate("nullable_float16_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 2); - EXPECT_EQ(schema.type_descriptor.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(schema.type_descriptor.is_string_like); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_FLOAT); - const auto& nullable_column = assert_cast(column); - const auto& values = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FLOAT_EQ(values.get_element(1), 0.0F); - EXPECT_FALSE(std::signbit(values.get_element(1))); - EXPECT_FLOAT_EQ(values.get_element(2), -0.0F); - EXPECT_TRUE(std::signbit(values.get_element(2))); - EXPECT_FLOAT_EQ(values.get_element(3), 1.5F); - EXPECT_TRUE(std::isnan(values.get_element(4))); - }); - read_and_validate("binary_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BYTE_ARRAY); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "bin_a"); - EXPECT_EQ(values.get_data_at(3).to_string(), "bin_d"); - }); - read_and_validate("string_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - read_and_validate("fixed_binary_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 4); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "aaaa"); - EXPECT_EQ(values.get_data_at(2).to_string(), "cccc"); - }); - read_and_validate("date_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATEV2); - EXPECT_EQ(schema.type->to_string(column, 0), "1970-01-01"); - EXPECT_EQ(schema.type->to_string(column, 2), "2021-01-01"); - }); - read_and_validate( - "timestamp_millis_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999"); - }); - read_and_validate( - "timestamp_micros_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("timestamp_micros_utc_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.timestamp_is_adjusted_to_utc); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("decimal_fixed_binary_9_2_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal32(12345)); - EXPECT_EQ(schema.type->to_string(column, 0), "123.45"); - }); - read_and_validate("decimal_fixed_binary_18_6_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal64(1234567)); - EXPECT_EQ(schema.type->to_string(column, 0), "1.234567"); - }); - read_and_validate( - "nullable_int_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); -} - -TEST_F(ParquetSerdeReaderTest, ReadInt96TimestampAsDateTimeV2) { - const auto file_path = (_test_dir / "int96_timestamp.parquet").string(); - auto field = arrow::field("col_datetime", arrow::timestamp(arrow::TimeUnit::MICRO), false); - auto array = build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1}); - auto table = arrow::Table::Make(arrow::schema({field}), {array}); - - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_builder.enable_force_write_int96_timestamps(); - _fields.clear(); - _file_reader.reset(); - _row_group.reset(); - write_table(file_path, table, arrow_builder.build()); - open_file(file_path); - - ASSERT_EQ(_fields.size(), 1); - EXPECT_EQ(_fields[0]->type_descriptor.physical_type, ::parquet::Type::INT96); - EXPECT_EQ(_fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - ASSERT_TRUE(supports_record_reader(_fields[0]->type_descriptor)); - ASSERT_EQ(remove_nullable(_fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - - auto reader = create_reader(0); - ASSERT_NE(reader, nullptr); - auto column = _fields[0]->type->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(ROW_COUNT, column, &rows_read).ok()); - ASSERT_EQ(rows_read, ROW_COUNT); - EXPECT_EQ(_fields[0]->type->to_string(*column, 0), "1970-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 2), "2021-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 4), "1969-12-31 23:59:59.999999"); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index dd2138279b11d1..e69f1311e4ab2b 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -17,29 +17,26 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include #include -#include -#include -#include -#include -#include #include #include #include +#include #include #include #include #include +#include #include #include +#include "core/data_type/data_type_date.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_time.h" #include "core/field.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr.h" @@ -47,293 +44,157 @@ #include "exprs/vslot_ref.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "storage/index/bloom_filter/block_split_bloom_filter.h" -#include "storage/index/zone_map/zonemap_eval_context.h" -#include "storage/index/zone_map/zonemap_filter_result.h" - +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" +#include "io/fs/file_reader.h" +#include "util/thrift_util.h" namespace doris { namespace { -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -std::shared_ptr int32_array(const std::vector>& values) { - arrow::Int32Builder builder; - for (const auto& value : values) { - if (value.has_value()) { - EXPECT_TRUE(builder.Append(*value).ok()); - } else { - EXPECT_TRUE(builder.AppendNull().ok()); - } - } - return finish_array(&builder); -} - -std::shared_ptr uint32_array(const std::vector& values) { - arrow::UInt32Builder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr float_array(const std::vector& values) { - arrow::FloatBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr double_array(const std::vector& values) { - arrow::DoubleBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -template -std::string encoded_value(const NativeType& value) { - return {reinterpret_cast(&value), sizeof(value)}; -} - -std::shared_ptr string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr timestamp_array(const std::vector& values) { - arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::unique_ptr<::parquet::ParquetFileReader> make_reader( - const std::shared_ptr& table, int64_t row_group_size, bool enable_dictionary, - bool enable_statistics) { - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.compression(::parquet::Compression::UNCOMPRESSED); - if (enable_dictionary) { - builder.enable_dictionary(); - } else { - builder.disable_dictionary(); - } - if (!enable_statistics) { - builder.disable_statistics(); - } - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - row_group_size, builder.build()) - .ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - return ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); -} - -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*reader.metadata()->schema(), &file_schema) - .ok()); - return file_schema; -} - -template -class TestColumnIndex final : public ::parquet::TypedColumnIndex { +class StatisticsMemoryFileReader final : public io::FileReader { public: - using NativeType = typename ParquetDType::c_type; - - TestColumnIndex(NativeType min_value, NativeType max_value) - : TestColumnIndex(std::vector {min_value}, - std::vector {max_value}) {} - - TestColumnIndex(std::vector min_values, std::vector max_values) - : _null_pages(min_values.size(), false), - _null_counts(min_values.size(), 0), - _min_values(std::move(min_values)), - _max_values(std::move(max_values)) { - EXPECT_EQ(_min_values.size(), _max_values.size()); - for (size_t page_idx = 0; page_idx < _min_values.size(); ++page_idx) { - _non_null_page_indices.push_back(static_cast(page_idx)); - } - } + explicit StatisticsMemoryFileReader(std::vector bytes) + : _bytes(std::move(bytes)), _path("native-bloom-filter.parquet") {} - const std::vector& null_pages() const override { return _null_pages; } - const std::vector& encoded_min_values() const override { return _encoded_values; } - const std::vector& encoded_max_values() const override { return _encoded_values; } - ::parquet::BoundaryOrder::type boundary_order() const override { - return ::parquet::BoundaryOrder::Unordered; + Status close() override { + _closed = true; + return Status::OK(); } - bool has_null_counts() const override { return true; } - const std::vector& null_counts() const override { return _null_counts; } - const std::vector& non_null_page_indices() const override { - return _non_null_page_indices; + const io::Path& path() const override { return _path; } + size_t size() const override { return _bytes.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 1; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext*) override { + if (offset > _bytes.size() || result.size > _bytes.size() - offset) { + return Status::IOError("native Bloom test read exceeds memory file"); + } + memcpy(result.data, _bytes.data() + offset, result.size); + *bytes_read = result.size; + return Status::OK(); } - const std::vector& min_values() const override { return _min_values; } - const std::vector& max_values() const override { return _max_values; } private: - const std::vector _null_pages; - const std::vector _encoded_values; - const std::vector _null_counts; - std::vector _non_null_page_indices; - const std::vector _min_values; - const std::vector _max_values; + std::vector _bytes; + io::Path _path; + bool _closed = false; }; - -class Int32ZoneMapExpr final : public VExpr { +class BloomInExpr final : public VExpr { public: - enum class Op { GE, GT, IS_NULL, IS_NOT_NULL }; - - Int32ZoneMapExpr(int column_id, Op op, int32_t value = 0) + BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) : VExpr(std::make_shared(), false), - _column_id(column_id), - _op(op), - _value(value) {} + _slot(VSlotRef::create_shared(0, column_id, -1, std::move(data_type), "c0")), + _values(std::move(values)) {} const std::string& expr_name() const override { return _expr_name; } Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError("Int32ZoneMapExpr is only used by parquet statistics tests"); + return Status::InternalError("BloomInExpr is only used by parquet statistics tests"); } - bool can_evaluate_zonemap_filter() const override { return true; } + bool can_evaluate_bloom_filter() const override { return true; } - void collect_slot_column_ids(std::set& column_ids) const override { - column_ids.insert(_column_id); + ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { + return expr_zonemap::eval_in_bloom_filter(ctx, _slot, false, _values); } - ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { - auto zone_map = ctx.zone_map(_column_id); - if (zone_map == nullptr) { - return unsupported_zonemap_filter(ctx); - } - if (_op == Op::IS_NULL) { - return zone_map->has_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (_op == Op::IS_NOT_NULL) { - return zone_map->has_not_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (!zone_map->has_not_null) { - return ZoneMapFilterResult::kNoMatch; - } - const auto literal = Field::create_field(_value); - if (_op == Op::GE) { - return zone_map->max_value < literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; - } - return zone_map->max_value <= literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; + void collect_slot_column_ids(std::set& column_ids) const override { + _slot->collect_slot_column_ids(column_ids); } private: - int _column_id; - Op _op; - int32_t _value; - const std::string _expr_name = "Int32ZoneMapExpr"; + VExprSPtr _slot; + std::vector _values; + const std::string _expr_name = "BloomInExpr"; }; -class StringDictionaryInExpr final : public VExpr { +class DictionaryStringInExpr final : public VExpr { public: - StringDictionaryInExpr(int column_id, std::vector values) - : VExpr(std::make_shared(), false), - _slot(VSlotRef::create_shared(0, column_id, -1, std::make_shared(), - "c0")) { - _values.reserve(values.size()); - for (auto& value : values) { - _values.emplace_back(Field::create_field(std::move(value))); - } - } + DictionaryStringInExpr() : VExpr(std::make_shared(), false) {} const std::string& expr_name() const override { return _expr_name; } Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError( - "StringDictionaryInExpr is only used by parquet statistics tests"); + return Status::InternalError("DictionaryStringInExpr is metadata-only"); } bool can_evaluate_dictionary_filter() const override { return true; } - ZoneMapFilterResult evaluate_dictionary_filter( - const DictionaryEvalContext& ctx) const override { - return expr_zonemap::eval_in_dictionary(ctx, _slot, false, _values); + ZoneMapFilterResult evaluate_dictionary_filter(const DictionaryEvalContext&) const override { + return ZoneMapFilterResult::kNoMatch; } - void collect_slot_column_ids(std::set& column_ids) const override { - _slot->collect_slot_column_ids(column_ids); - } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } private: - VExprSPtr _slot; - std::vector _values; - const std::string _expr_name = "StringDictionaryInExpr"; + const std::string _expr_name = "DictionaryStringInExpr"; }; -class BloomInExpr final : public VExpr { +class MetadataInt32GreaterThanExpr final : public VExpr { public: - BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) - : VExpr(std::make_shared(), false), - _slot(VSlotRef::create_shared(0, column_id, -1, std::move(data_type), "c0")), - _values(std::move(values)) {} + explicit MetadataInt32GreaterThanExpr(int32_t value) + : VExpr(std::make_shared(), false), _value(value) {} const std::string& expr_name() const override { return _expr_name; } - Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, ColumnPtr&) const override { - return Status::InternalError("BloomInExpr is only used by parquet statistics tests"); - } - - bool can_evaluate_bloom_filter() const override { return true; } - - ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext& ctx) const override { - return expr_zonemap::eval_in_bloom_filter(ctx, _slot, false, _values); + return Status::InternalError("MetadataInt32GreaterThanExpr is metadata-only"); } - - void collect_slot_column_ids(std::set& column_ids) const override { - _slot->collect_slot_column_ids(column_ids); + bool can_evaluate_zonemap_filter() const override { return true; } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(0); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + return zone_map->max_value <= Field::create_field(_value) + ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; } private: - VExprSPtr _slot; - std::vector _values; - const std::string _expr_name = "BloomInExpr"; + int32_t _value; + const std::string _expr_name = "MetadataInt32GreaterThanExpr"; }; -format::FileScanRequest request_with_zonemap_conjunct(std::shared_ptr expr) { - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared(std::move(expr))); - return request; -} +class MetadataBoundsProbeExpr final : public VExpr { +public: + explicit MetadataBoundsProbeExpr(bool require_false_boolean = false) + : VExpr(std::make_shared(), false), + _require_false_boolean(require_false_boolean) {} -format::FileScanRequest request_with_dictionary_conjunct(std::vector values) { - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared( - std::make_shared(0, std::move(values)))); - return request; -} + const std::string& expr_name() const override { return _expr_name; } + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InternalError("MetadataBoundsProbeExpr is metadata-only"); + } + bool can_evaluate_zonemap_filter() const override { return true; } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(0); + if (zone_map == nullptr || !zone_map->has_not_null || zone_map->min_value.is_null() || + zone_map->max_value.is_null()) { + return ZoneMapFilterResult::kMayMatch; + } + if (_require_false_boolean && + zone_map->min_value == Field::create_field(false) && + zone_map->max_value == Field::create_field(false)) { + return ZoneMapFilterResult::kMayMatch; + } + return ZoneMapFilterResult::kNoMatch; + } +private: + bool _require_false_boolean; + const std::string _expr_name = "MetadataBoundsProbeExpr"; +}; VExprContextSPtrs bloom_conjuncts(DataTypePtr data_type, std::vector values) { return {VExprContext::create_shared( std::make_shared(0, std::move(data_type), std::move(values)))}; @@ -346,690 +207,514 @@ format::FileScanRequest request_with_bloom_conjunct(DataTypePtr data_type, request.conjuncts = bloom_conjuncts(std::move(data_type), std::move(values)); return request; } - -void add_bloom_field(segment_v2::BlockSplitBloomFilter* bloom_filter, const Field& value, - PrimitiveType type) { - DORIS_CHECK(bloom_filter != nullptr); - switch (type) { - case TYPE_BOOLEAN: { - const bool typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_INT: { - const int32_t typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_STRING: { - const auto& typed_value = value.get(); - bloom_filter->add_bytes(typed_value.data(), typed_value.size()); - break; - } - default: - DORIS_CHECK(false); - } -} - -std::unique_ptr bloom_filter_for_fields( - const std::vector& values, PrimitiveType type) { - auto bloom_filter = std::make_unique(); - EXPECT_TRUE(bloom_filter->init(segment_v2::BloomFilter::MINIMUM_BYTES).ok()); - for (const auto& value : values) { - add_bloom_field(bloom_filter.get(), value, type); - } - return bloom_filter; -} - -BloomFilterEvalContext bloom_context(const DataTypePtr& data_type, - const segment_v2::BloomFilter* bloom_filter) { - BloomFilterEvalContext ctx; - ctx.slots.emplace(0, BloomFilterEvalContext::SlotBloomFilter {.data_type = data_type, - .bloom_filter = bloom_filter}); - return ctx; -} - -std::unique_ptr<::parquet::BlockSplitBloomFilter> parquet_bloom_filter() { - auto bloom_filter = std::make_unique<::parquet::BlockSplitBloomFilter>(); - bloom_filter->Init(::parquet::BlockSplitBloomFilter::kMinimumBloomFilterBytes); - return bloom_filter; -} - format::parquet::ParquetColumnSchema uint32_parquet_bloom_schema() { format::parquet::ParquetColumnSchema column_schema; column_schema.type = std::make_shared(); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::INT32; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; column_schema.type_descriptor.integer_bit_width = 32; column_schema.type_descriptor.is_unsigned_integer = true; return column_schema; } -format::parquet::ParquetColumnSchema fixed_len_string_parquet_bloom_schema(int fixed_length) { +TEST(NativeParquetStatisticsTest, InvalidNullableDateBoundsDisableMinMax) { format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); + column_schema.type = make_nullable(std::make_shared()); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = fixed_length; - column_schema.type_descriptor.is_string_like = true; - return column_schema; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; + + const int32_t invalid_date = std::numeric_limits::min(); + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value( + std::string(reinterpret_cast(&invalid_date), sizeof(invalid_date))); + statistics.__set_max_value( + std::string(reinterpret_cast(&invalid_date), sizeof(invalid_date))); + + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + EXPECT_FALSE(result.has_min_max); } -format::parquet::ParquetColumnSchema float16_parquet_bloom_schema() { +TEST(NativeParquetStatisticsTest, InvalidNullableDecimalBoundsDisableMinMax) { format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); + column_schema.type = make_nullable(std::make_shared(2, 0)); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = 2; - column_schema.type_descriptor.extra_type_info = format::parquet::ParquetExtraTypeInfo::FLOAT16; - return column_schema; -} - -TEST(ParquetStatisticsTransformTest, ConvertsMinMaxNullCountUnsignedStringAndTimestamp) { - auto table = arrow::Table::Make( - arrow::schema({ - arrow::field("i", arrow::int32(), true), - arrow::field("u", arrow::uint32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - }), - {int32_array({1, std::nullopt, 5}), uint32_array({7, 9, 11}), - string_array({"alpha", "beta", "omega"}), timestamp_array({1000, 2000, 3000})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto row_group = reader->metadata()->RowGroup(0); - - const auto int_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], row_group->ColumnChunk(0)->statistics()); - EXPECT_TRUE(int_stats.has_min_max); - EXPECT_TRUE(int_stats.has_null_count); - EXPECT_TRUE(int_stats.has_null); - EXPECT_TRUE(int_stats.has_not_null); - EXPECT_EQ(int_stats.min_value.get(), 1); - EXPECT_EQ(int_stats.max_value.get(), 5); - - const auto uint_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[1], row_group->ColumnChunk(1)->statistics()); - EXPECT_TRUE(uint_stats.has_min_max); - EXPECT_EQ(uint_stats.min_value.get(), 7); - EXPECT_EQ(uint_stats.max_value.get(), 11); - - const auto string_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], row_group->ColumnChunk(2)->statistics()); - EXPECT_TRUE(string_stats.has_min_max); - EXPECT_EQ(string_stats.min_value.get(), "alpha"); - EXPECT_EQ(string_stats.max_value.get(), "omega"); - - auto utc = cctz::utc_time_zone(); - const auto timestamp_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[3], row_group->ColumnChunk(3)->statistics(), &utc); - EXPECT_TRUE(timestamp_stats.has_min_max); - EXPECT_EQ(timestamp_stats.min_value.get_type(), TYPE_DATETIMEV2); - EXPECT_EQ(timestamp_stats.max_value.get_type(), TYPE_DATETIMEV2); - EXPECT_LT(timestamp_stats.min_value, timestamp_stats.max_value); -} + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; + column_schema.type_descriptor.is_decimal = true; + column_schema.type_descriptor.decimal_precision = 2; + column_schema.type_descriptor.decimal_scale = 0; -TEST(ParquetStatisticsTransformTest, DisablesUtcTimestampMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - // America/New_York moved from UTC-04:00 to UTC-05:00 at 2021-11-07 06:00:00 UTC. - // Both UTC endpoints below map to 01:30 local time, while values inside the interval cover - // 01:00 through 01:59. Endpoint conversion therefore cannot represent the true local range. - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636263900 * MICROS_PER_SECOND, - 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto local_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(local_stats.has_not_null); - EXPECT_FALSE(local_stats.has_min_max); - - auto utc = cctz::utc_time_zone(); - const auto utc_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &utc); - EXPECT_TRUE(utc_stats.has_min_max); - EXPECT_LT(utc_stats.min_value, utc_stats.max_value); -} + const int32_t invalid_decimal = 1000; + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value( + std::string(reinterpret_cast(&invalid_decimal), sizeof(invalid_decimal))); + statistics.__set_max_value( + std::string(reinterpret_cast(&invalid_decimal), sizeof(invalid_decimal))); -TEST(ParquetStatisticsTransformTest, KeepsTimestampTzMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - // This is the effective type produced by enable_mapping_timestamp_tz. The physical timestamp - // flags intentionally remain adjusted-to-UTC so decoding can preserve the source semantics. - schema[0]->type = std::make_shared(6); - schema[0]->type_descriptor.doris_type = schema[0]->type; - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto timestamp_tz_stats = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(timestamp_tz_stats.has_min_max); - EXPECT_EQ(timestamp_tz_stats.min_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_EQ(timestamp_tz_stats.max_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_LT(timestamp_tz_stats.min_value, timestamp_tz_stats.max_value); + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + EXPECT_FALSE(result.has_min_max); } -TEST(ParquetStatisticsTransformTest, HandlesMissingStatisticsAndAllNullChunks) { - auto no_stats_table = arrow::Table::Make( - arrow::schema({arrow::field("i", arrow::int32(), true)}), {int32_array({1, 2, 3})}); - auto no_stats_reader = make_reader(no_stats_table, 3, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - auto no_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *no_stats_schema[0], - no_stats_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_FALSE(no_stats.has_min_max); - - auto all_null_table = - arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt})}); - auto all_null_reader = make_reader(all_null_table, 2, false, true); - auto all_null_schema = build_file_schema(*all_null_reader); - auto all_null_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *all_null_schema[0], - all_null_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_TRUE(all_null_stats.has_null_count); - EXPECT_TRUE(all_null_stats.has_null); - EXPECT_FALSE(all_null_stats.has_not_null); - EXPECT_FALSE(all_null_stats.has_min_max); -} +TEST(NativeParquetStatisticsTest, InvalidTimeBoundsDisableFooterMinMax) { + format::parquet::ParquetColumnSchema column_schema; + column_schema.type = std::make_shared(3); + column_schema.type_descriptor.doris_type = column_schema.type; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; + column_schema.type_descriptor.time_unit = format::parquet::ParquetTimeUnit::MILLIS; -TEST(ParquetStatisticsTransformTest, MissingNullCountConservativelyReportsPossibleNulls) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({1, std::nullopt, 3})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto file_statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - auto statistics_without_null_count = ::parquet::MakeStatistics<::parquet::Int32Type>( - reader->metadata()->schema()->Column(0), file_statistics->EncodeMin(), - file_statistics->EncodeMax(), file_statistics->num_values(), 0, 0, true, false, false); - - const auto statistics = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics_without_null_count); - EXPECT_FALSE(statistics.has_null_count); - EXPECT_TRUE(statistics.has_null); - EXPECT_TRUE(statistics.has_not_null); - EXPECT_TRUE(statistics.has_min_max); - EXPECT_EQ(statistics.min_value.get(), 1); - EXPECT_EQ(statistics.max_value.get(), 3); -} + const int32_t invalid_time = 90000000; + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value( + std::string(reinterpret_cast(&invalid_time), sizeof(invalid_time))); + statistics.__set_max_value( + std::string(reinterpret_cast(&invalid_time), sizeof(invalid_time))); -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const float float_nan = std::numeric_limits::quiet_NaN(); - const float float_max = 2.0F; - auto float_stats = ::parquet::MakeStatistics<::parquet::FloatType>( - schema[0]->descriptor, encoded_value(float_nan), encoded_value(float_max), 2, 0, 0, - true, true, false); - const auto converted_float = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], float_stats); - EXPECT_FALSE(converted_float.has_min_max); - EXPECT_TRUE(converted_float.has_not_null); - - const double double_nan = std::numeric_limits::quiet_NaN(); - const double double_min = 1.0; - auto double_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_nan), 2, 0, 0, - true, true, false); - const auto converted_double = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - double_stats); - EXPECT_FALSE(converted_double.has_min_max); - EXPECT_TRUE(converted_double.has_not_null); - - const double double_max = 2.0; - auto finite_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_max), 2, 0, 0, - true, true, false); - const auto converted_finite = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - finite_stats); - EXPECT_TRUE(converted_finite.has_min_max); + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + EXPECT_FALSE(result.has_min_max); } -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleColumnIndexMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - auto float_index = std::make_shared>( - 1.0F, std::numeric_limits::quiet_NaN()); - format::parquet::ParquetColumnStatistics float_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - float_index, *schema[0], 0, &float_page_stats)); - EXPECT_FALSE(float_page_stats.has_min_max); - EXPECT_TRUE(float_page_stats.has_not_null); - - auto double_index = std::make_shared>( - std::numeric_limits::quiet_NaN(), 2.0); - format::parquet::ParquetColumnStatistics double_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - double_index, *schema[1], 0, &double_page_stats)); - EXPECT_FALSE(double_page_stats.has_min_max); - EXPECT_TRUE(double_page_stats.has_not_null); - - auto finite_index = std::make_shared>(1.0, 2.0); - format::parquet::ParquetColumnStatistics finite_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - finite_index, *schema[1], 0, &finite_page_stats)); - EXPECT_TRUE(finite_page_stats.has_min_max); - - auto mixed_index = std::make_shared>( - std::vector {std::numeric_limits::quiet_NaN(), 1.0}, - std::vector {std::numeric_limits::quiet_NaN(), 2.0}); - format::parquet::ParquetColumnStatistics nan_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 0, &nan_page_stats)); - EXPECT_FALSE(nan_page_stats.has_min_max); - - format::parquet::ParquetColumnStatistics following_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 1, &following_page_stats)); - EXPECT_TRUE(following_page_stats.has_min_max); - EXPECT_EQ(following_page_stats.min_value.get(), 1.0); - EXPECT_EQ(following_page_stats.max_value.get(), 2.0); -} +TEST(NativeParquetStatisticsTest, BooleanFooterBoundsDecodeOnlyTheValueBit) { + format::parquet::ParquetColumnSchema column_schema; + column_schema.type = std::make_shared(); + column_schema.type_descriptor.doris_type = column_schema.type; + column_schema.type_descriptor.physical_type = tparquet::Type::BOOLEAN; + + tparquet::Statistics statistics; + statistics.__set_null_count(0); + statistics.__set_min_value(std::string(1, '\x02')); + statistics.__set_max_value(std::string(1, '\x02')); + + const auto result = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + column_schema, &statistics, 1, nullptr); + ASSERT_TRUE(result.has_min_max); + EXPECT_EQ(result.min_value, Field::create_field(false)); + EXPECT_EQ(result.max_value, Field::create_field(false)); +} + +TEST(NativeParquetStatisticsTest, InvalidTimeAndPaddedBooleanPageBoundsCannotPrune) { + auto run_page_index = [](std::unique_ptr column_schema, + std::string min_value, std::string max_value, + bool require_false_boolean) { + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + tparquet::FileMetaData metadata; + metadata.__set_column_orders({order}); + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = {VExprContext::create_shared( + std::make_shared(require_false_boolean))}; + + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_min_values({std::move(min_value)}); + page_index.column_index.__set_max_values({std::move(max_value)}); + page_index.column_index.__set_null_pages({false}); + page_index.column_index.__set_null_counts({0}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, std::move(page_index)); + std::vector selected_ranges; + std::map skip_plans; + EXPECT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, + &skip_plans, nullptr) + .ok()); + return selected_ranges; + }; + + auto time_schema = std::make_unique(); + time_schema->type = std::make_shared(3); + time_schema->type_descriptor.doris_type = time_schema->type; + time_schema->type_descriptor.physical_type = tparquet::Type::INT32; + time_schema->type_descriptor.time_unit = format::parquet::ParquetTimeUnit::MILLIS; + const int32_t invalid_time = 90000000; + const std::string invalid_time_bytes(reinterpret_cast(&invalid_time), + sizeof(invalid_time)); + const auto time_ranges = + run_page_index(std::move(time_schema), invalid_time_bytes, invalid_time_bytes, false); + ASSERT_EQ(time_ranges.size(), 1); + EXPECT_EQ(time_ranges[0].start, 0); + EXPECT_EQ(time_ranges[0].length, 1); + + auto bool_schema = std::make_unique(); + bool_schema->type = std::make_shared(); + bool_schema->type_descriptor.doris_type = bool_schema->type; + bool_schema->type_descriptor.physical_type = tparquet::Type::BOOLEAN; + const auto bool_ranges = run_page_index(std::move(bool_schema), std::string(1, '\x02'), + std::string(1, '\x02'), true); + ASSERT_EQ(bool_ranges.size(), 1); + EXPECT_EQ(bool_ranges[0].start, 0); + EXPECT_EQ(bool_ranges[0].length, 1); +} +TEST(ParquetBloomFilterPruningTest, NativeUint32BloomUsesPhysicalInt32Hash) { + const auto column_schema = uint32_parquet_bloom_schema(); + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) + .ok()); -TEST(ParquetStatisticsTransformTest, IgnoresInvertedFooterAndColumnIndexMinMax) { - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("i", arrow::int32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {int32_array({1, 2}), string_array({"a", "z"}), timestamp_array({1000000, 2000000})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const int32_t inverted_min = 10; - const int32_t inverted_max = 1; - auto int_stats = ::parquet::MakeStatistics<::parquet::Int32Type>( - schema[0]->descriptor, encoded_value(inverted_min), encoded_value(inverted_max), 2, 0, - 0, true, true, false); - const auto converted_int = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], int_stats); - EXPECT_TRUE(converted_int.has_not_null); - EXPECT_FALSE(converted_int.has_min_max); - - const std::string inverted_string_min = "z"; - const std::string inverted_string_max = "a"; - auto string_stats = ::parquet::MakeStatistics<::parquet::ByteArrayType>( - schema[1]->descriptor, inverted_string_min, inverted_string_max, 2, 0, 0, true, true, - false); - const auto converted_string = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - string_stats); - EXPECT_TRUE(converted_string.has_not_null); - EXPECT_FALSE(converted_string.has_min_max); - - auto int_index = - std::make_shared>(inverted_min, inverted_max); - format::parquet::ParquetColumnStatistics page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - int_index, *schema[0], 0, &page_stats)); - EXPECT_TRUE(page_stats.has_not_null); - EXPECT_FALSE(page_stats.has_min_max); - - // These endpoints are inverted within one second. Whole-second validation alone would miss - // the corruption, and TIMESTAMPTZ must reject the same raw inversion before its UTC shortcut. - constexpr int64_t timestamp_min = 1500000; - constexpr int64_t timestamp_max = 1000000; - auto timestamp_stats = ::parquet::MakeStatistics<::parquet::Int64Type>( - schema[2]->descriptor, encoded_value(timestamp_min), encoded_value(timestamp_max), 2, 0, - 0, true, true, false); - auto utc = cctz::utc_time_zone(); - const auto converted_timestamp = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp.has_min_max); - - schema[2]->type = std::make_shared(6); - schema[2]->type_descriptor.doris_type = schema[2]->type; - const auto converted_timestamp_tz = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp_tz.has_min_max); -} + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); -TEST(ParquetStatisticsTransformTest, PreservesNullCountWhenNaNInvalidatesMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float64(), false)}), - {double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const double nan = std::numeric_limits::quiet_NaN(); - const double max_value = 2.0; - auto footer_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[0]->descriptor, encoded_value(nan), encoded_value(max_value), 2, 0, 0, true, - true, false); - const auto converted_footer = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[0], - footer_stats); - auto footer_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(converted_footer); - ASSERT_NE(footer_zone_map, nullptr); - EXPECT_TRUE(footer_zone_map->pass_all); - EXPECT_FALSE(footer_zone_map->has_null); - EXPECT_TRUE(footer_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*footer_zone_map, schema[0]->type)); - - ZoneMapEvalContext footer_ctx; - footer_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = footer_zone_map}); - Int32ZoneMapExpr is_null_expr(0, Int32ZoneMapExpr::Op::IS_NULL); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(footer_ctx), ZoneMapFilterResult::kNoMatch); - - auto column_index = std::make_shared>(nan, max_value); - format::parquet::ParquetColumnStatistics page_stats; - ASSERT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *schema[0], 0, &page_stats)); - auto page_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(page_stats); - ASSERT_NE(page_zone_map, nullptr); - EXPECT_TRUE(page_zone_map->pass_all); - EXPECT_FALSE(page_zone_map->has_null); - EXPECT_TRUE(page_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*page_zone_map, schema[0]->type)); - - ZoneMapEvalContext page_ctx; - page_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = page_zone_map}); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(page_ctx), ZoneMapFilterResult::kNoMatch); + EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field( + static_cast(present_value))}), + bloom_filter)); + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), + bloom_filter)); } -TEST(ParquetStatisticsPruningTest, ExprZonemapPredicatesAndNullPredicatesPruneRowGroups) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt, 3, 4, 5, 6})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); +TEST(ParquetBloomFilterPruningTest, NativeRowGroupKeepsPresentUint32AboveInt32Max) { + auto column_schema = + std::make_unique(uint32_parquet_bloom_schema()); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct( - std::make_shared(0, Int32ZoneMapExpr::Op::GE, 5)), - nullptr, &selected, false, &pruning_stats) + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) .ok()); - EXPECT_EQ(selected, std::vector({2})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 2); - - selected.clear(); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NOT_NULL)), - nullptr, &selected, false, &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({1, 2})); - - selected.clear(); + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); + + tparquet::BloomFilterAlgorithm algorithm; + algorithm.__set_BLOCK(tparquet::SplitBlockAlgorithm()); + tparquet::BloomFilterHash hash; + hash.__set_XXHASH(tparquet::XxHash()); + tparquet::BloomFilterCompression compression; + compression.__set_UNCOMPRESSED(tparquet::Uncompressed()); + tparquet::BloomFilterHeader bloom_header; + bloom_header.__set_numBytes(static_cast(bloom_filter.size())); + bloom_header.__set_algorithm(algorithm); + bloom_header.__set_hash(hash); + bloom_header.__set_compression(compression); + std::vector bloom_bytes; + ThriftSerializer serializer(/*compact=*/true, 64); + ASSERT_TRUE(serializer.serialize(&bloom_header, &bloom_bytes).ok()); + bloom_bytes.insert(bloom_bytes.end(), bloom_filter.data(), + bloom_filter.data() + bloom_filter.size()); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(1); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_bloom_filter_offset(0); + column_metadata.__set_bloom_filter_length(static_cast(bloom_bytes.size())); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_version(1); + metadata.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + + format::parquet::ParquetFileContext file_context; + file_context.native_file = std::make_shared(std::move(bloom_bytes)); + auto request = request_with_bloom_conjunct( + column_schema->type, + {Field::create_field(static_cast(present_value))}); + std::vector> schema; + schema.push_back(std::move(column_schema)); + std::vector selected_row_groups; + format::parquet::ParquetPruningStats pruning_stats; ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NULL)), - nullptr, &selected, false, &pruning_stats) + metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) .ok()); - EXPECT_EQ(selected, std::vector({0})); + EXPECT_EQ(selected_row_groups, std::vector({0})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); } -TEST(ParquetStatisticsPruningTest, DictionaryPruningHandlesExcludeIncludeAndUnsupportedPaths) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, false); - auto schema = build_file_schema(*reader); +TEST(NativeParquetStatisticsTest, EmptyDictionaryRowGroupIsSkippedBeforeMetadataProbes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::BYTE_ARRAY); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(0); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_dictionary_page_offset(0); + column_metadata.__set_encodings({tparquet::Encoding::RLE_DICTIONARY}); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(0); + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root, leaf}); + thrift_metadata.__set_num_rows(0); + thrift_metadata.__set_row_groups({row_group}); + + format::parquet::NativeParquetMetadata native_metadata(thrift_metadata, 0); + ASSERT_TRUE(native_metadata.init_schema(false, false).ok()); + format::parquet::ParquetFileContext file_context; + file_context.native_file = + std::make_shared(std::vector {}); + file_context.native_metadata = &native_metadata; + + auto column_schema = std::make_unique(); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::BYTE_ARRAY; + column_schema->type_descriptor.is_string_like = true; + std::vector> schema; + schema.push_back(std::move(column_schema)); - std::vector selected; + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = {VExprContext::create_shared(std::make_shared())}; + std::vector selected_row_groups; format::parquet::ParquetPruningStats pruning_stats; ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"gamma"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 1); - - auto plain_reader = make_reader(table, 2, false, false); - auto plain_schema = build_file_schema(*plain_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *plain_reader->metadata(), plain_reader.get(), plain_schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) + thrift_metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 0); + EXPECT_TRUE(selected_row_groups.empty()); } -TEST(ParquetStatisticsPruningTest, VExprUsesDictionaryAndMissingBloomKeepsRows) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, true); - auto schema = build_file_schema(*reader); +TEST(NativeParquetStatisticsTest, InvalidCandidateRowGroupReturnsCorruption) { + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); + format::FileScanRequest request; + const std::vector> schema; + const std::vector candidates {1}; + std::vector selected_row_groups; + + const auto status = format::parquet::select_row_groups_by_metadata( + metadata, schema, request, &candidates, &selected_row_groups, false, nullptr, nullptr, + nullptr, nullptr); + EXPECT_TRUE(status.is()) << status; +} +TEST(NativeParquetStatisticsTest, LegacyBinaryFooterBoundsRequireComparableOrdering) { + format::parquet::ParquetTypeDescriptor binary_type; + binary_type.physical_type = tparquet::Type::BYTE_ARRAY; + + tparquet::Statistics max_only; + max_only.__set_max("III"); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(binary_type, max_only, false)); + + tparquet::Statistics legacy_different; + legacy_different.__set_min("III"); + legacy_different.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, + legacy_different, false)); + + tparquet::Statistics legacy_equal; + legacy_equal.__set_min("same"); + legacy_equal.__set_max("same"); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, legacy_equal, + false)); + + tparquet::Statistics type_defined; + type_defined.__set_min_value("III"); + type_defined.__set_max_value("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + false)); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + true)); + + tparquet::Statistics mixed_fields; + mixed_fields.__set_min_value("III"); + mixed_fields.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, mixed_fields, + true)); +} + +TEST(NativeParquetStatisticsTest, ExplicitlyInexactNumericBoundsCannotBackMinMaxAggregate) { + format::parquet::ParquetTypeDescriptor int32_type; + int32_type.physical_type = tparquet::Type::INT32; + auto encode_int32 = [](int32_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + + tparquet::Statistics statistics; + statistics.__set_min_value(encode_int32(0)); + statistics.__set_max_value(encode_int32(100)); + EXPECT_TRUE( + format::parquet::detail::can_use_native_footer_min_max(int32_type, statistics, true)); + + statistics.__set_is_min_value_exact(false); + statistics.__set_is_max_value_exact(true); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(int32_type, statistics, true)); + + statistics.__set_is_min_value_exact(true); + statistics.__set_is_max_value_exact(false); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(int32_type, statistics, true)); +} + +TEST(NativeParquetStatisticsTest, TypeDefinedBoundsRequireSupportedColumnOrder) { + auto encode_int32 = [](int32_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + + auto column_schema = std::make_unique(); + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::INT32; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::Statistics statistics; + statistics.__set_min_value(encode_int32(1)); + statistics.__set_max_value(encode_int32(2)); + statistics.__set_null_count(0); + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_num_values(1); + column_metadata.__set_statistics(statistics); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = { + VExprContext::create_shared(std::make_shared(100))}; + std::vector selected_row_groups; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - auto no_stats_reader = make_reader(table, 2, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *no_stats_reader->metadata(), no_stats_reader.get(), no_stats_schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) + EXPECT_EQ(selected_row_groups, std::vector({0})); + + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_min_values({encode_int32(1)}); + page_index.column_index.__set_max_values({encode_int32(2)}); + page_index.column_index.__set_null_pages({false}); + page_index.column_index.__set_null_counts({0}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, page_index); + std::vector selected_ranges; + std::map skip_plans; + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); -} - -TEST(ParquetStatisticsPruningTest, BloomFilterCacheIsScopedToRowGroupAndColumn) { - auto input = arrow::io::ReadableFile::Open( - "./be/test/exec/test_data/parquet_scanner/multi_row_group_bloom_filter.parquet"); - ASSERT_TRUE(input.ok()); - auto reader = ::parquet::ParquetFileReader::Open(*input); - ASSERT_EQ(reader->metadata()->num_row_groups(), 2); - auto& bloom_filter_reader = reader->GetBloomFilterReader(); - for (int row_group_idx = 0; row_group_idx < 2; ++row_group_idx) { - auto row_group_reader = bloom_filter_reader.RowGroup(row_group_idx); - ASSERT_NE(row_group_reader, nullptr); - ASSERT_NE(row_group_reader->GetColumnBloomFilter(0), nullptr); - } - auto schema = build_file_schema(*reader); - - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - auto request = request_with_bloom_conjunct(std::make_shared(), - {Field::create_field(12345)}); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - false, &pruning_stats) + EXPECT_EQ(selected_ranges.size(), 1); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + metadata.__set_column_orders({order}); + selected_row_groups.clear(); + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - true, &pruning_stats) + EXPECT_TRUE(selected_row_groups.empty()); + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 1); -} - -TEST(ParquetBloomFilterPruningTest, VExprEqPrunesAbsentIntValue) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2)}), ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(3)}), ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprInPrunesOnlyWhenAllValuesAreAbsent) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(4)}), - ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(3)}), - ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprBoolAndStringUseSlotBloomFilter) { - auto bool_type = std::make_shared(); - auto bool_filter = - bloom_filter_for_fields({Field::create_field(true)}, TYPE_BOOLEAN); - auto bool_ctx = bloom_context(bool_type, bool_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(false)}), - bool_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(true)}), - bool_ctx), - ZoneMapFilterResult::kMayMatch); - - auto string_type = std::make_shared(); - auto string_filter = bloom_filter_for_fields( - {Field::create_field("alpha"), Field::create_field("omega")}, - TYPE_STRING); - auto string_ctx = bloom_context(string_type, string_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("beta")}), - string_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("alpha")}), - string_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, MissingOrUnsupportedBloomContextKeepsRowGroup) { - auto int_type = std::make_shared(); - BloomFilterEvalContext missing_ctx; - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(int_type, {Field::create_field(2)}), missing_ctx), - ZoneMapFilterResult::kMayMatch); - - auto smallint_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields({Field::create_field(1)}, TYPE_INT); - auto unsupported_ctx = bloom_context(smallint_type, bloom_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(smallint_type, {Field::create_field(2)}), - unsupported_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, ParquetUint32BloomUsesPhysicalInt32Hash) { - const auto column_schema = uint32_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - const uint32_t present_value = 4000000000U; - int32_t physical_value; - memcpy(&physical_value, &present_value, sizeof(physical_value)); - bloom_filter->InsertHash(bloom_filter->Hash(physical_value)); - - // UINT32 is exposed to VExpr as Doris BIGINT, but Parquet stores and hashes it as a physical - // INT32 carrier. A present value above INT32_MAX must therefore be narrowed to the physical - // bit pattern before probing the file bloom filter. - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field( - static_cast(present_value))}), - *bloom_filter)); - - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts( - column_schema.type, - {Field::create_field( - static_cast(std::numeric_limits::max()) + 1)}), - *bloom_filter)); -} - -TEST(ParquetBloomFilterPruningTest, ParquetFixedLenByteArrayBloomUsesFlbaHash) { - const auto column_schema = fixed_len_string_parquet_bloom_schema(4); - auto bloom_filter = parquet_bloom_filter(); - - const std::string present_value = "abcd"; - ::parquet::FLBA physical_value(reinterpret_cast(present_value.data())); - bloom_filter->InsertHash( - bloom_filter->Hash(&physical_value, column_schema.type_descriptor.fixed_length)); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(present_value)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field("abc")}), - *bloom_filter)); -} - -TEST(ParquetBloomFilterPruningTest, ParquetFloat16BloomDoesNotUseFloatHash) { - const auto column_schema = float16_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(1.0F)}), - *bloom_filter)); + EXPECT_TRUE(selected_ranges.empty()); +} + +TEST(NativeParquetStatisticsTest, ContradictoryAllNullPageCountsDisablePruning) { + auto column_schema = std::make_unique(); + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::INT32; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + tparquet::FileMetaData metadata; + metadata.__set_column_orders({order}); + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = { + VExprContext::create_shared(std::make_shared(0))}; + + for (const int64_t contradictory_null_count : {5, 11}) { + SCOPED_TRACE(contradictory_null_count); + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_null_pages({true}); + page_index.column_index.__set_null_counts({contradictory_null_count}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, std::move(page_index)); + std::vector selected_ranges; + std::map skip_plans; + + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 10, &selected_ranges, + &skip_plans, nullptr) + .ok()); + // ColumnIndex is optional. An impossible all-null claim must fall back to reading the + // ten-row data page instead of proving that no value can satisfy the predicate. + ASSERT_EQ(selected_ranges.size(), 1); + EXPECT_EQ(selected_ranges[0].start, 0); + EXPECT_EQ(selected_ranges[0].length, 10); + } } } // namespace diff --git a/be/test/format_v2/parquet/parquet_type_test.cpp b/be/test/format_v2/parquet/parquet_type_test.cpp index 4bca77c1803b49..0680cc828a9f7a 100644 --- a/be/test/format_v2/parquet/parquet_type_test.cpp +++ b/be/test/format_v2/parquet/parquet_type_test.cpp @@ -17,478 +17,49 @@ #include "format_v2/parquet/parquet_type.h" -#include -#include #include -#include -#include -#include - -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/primitive_type.h" namespace doris::format::parquet { -namespace { - -::parquet::SchemaDescriptor make_descriptor(const ::parquet::schema::NodePtr& node) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, {node}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - return descriptor; -} - -ParquetTypeDescriptor resolve_node(const ::parquet::schema::NodePtr& node) { - auto descriptor = make_descriptor(node); - return resolve_parquet_type(descriptor.Column(0)); -} - -PrimitiveType primitive_type(const DataTypePtr& type) { - return remove_nullable(type)->get_primitive_type(); -} - -int scale_of(const DataTypePtr& type) { - return remove_nullable(type)->get_scale(); -} - -std::shared_ptr make_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.Append(0x3E00).ok()); - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetTypeDescriptor resolve_arrow_float16_type() { - const auto schema = arrow::schema({arrow::field("f16", arrow::float16(), true)}); - const auto table = arrow::Table::Make(schema, {make_float16_array()}); - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1).ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - - auto reader = ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); - return resolve_parquet_type(reader->metadata()->schema()->Column(0)); -} - -} // namespace - -TEST(ParquetTypeTest, ResolveLogicalIntegerMappings) { - struct Case { - int bit_width; - bool is_signed; - PrimitiveType expected_type; - bool expected_unsigned; - }; - const std::vector cases = { - {8, true, TYPE_TINYINT, false}, {8, false, TYPE_SMALLINT, true}, - {16, true, TYPE_SMALLINT, false}, {16, false, TYPE_INT, true}, - {32, true, TYPE_INT, false}, {32, false, TYPE_BIGINT, true}, - {64, true, TYPE_BIGINT, false}, {64, false, TYPE_LARGEINT, true}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.bit_width); - const auto node = ::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Int(test_case.bit_width, test_case.is_signed), - test_case.bit_width == 64 ? ::parquet::Type::INT64 : ::parquet::Type::INT32); - const auto type = resolve_node(node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_TRUE(type.supports_record_reader); - } -} - -TEST(ParquetTypeTest, ResolveLogicalTimeAndTimestampMappings) { - const auto time_millis = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - ASSERT_NE(time_millis.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_millis.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_millis.time_unit, ParquetTimeUnit::MILLIS); - EXPECT_EQ(time_millis.extra_type_info, ParquetExtraTypeInfo::UNIT_MS); - - const auto time_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_us", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64)); - ASSERT_NE(time_micros.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_micros.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(time_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - - const auto adjusted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_adjusted", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - EXPECT_EQ(adjusted_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_time.supports_record_reader); - EXPECT_FALSE(adjusted_time.unsupported_reason.empty()); - - const auto timestamp_nanos = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ns", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(timestamp_nanos.doris_type, nullptr); - EXPECT_TRUE(timestamp_nanos.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(timestamp_nanos.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(timestamp_nanos.is_timestamp); - EXPECT_TRUE(timestamp_nanos.timestamp_is_adjusted_to_utc); - EXPECT_EQ(timestamp_nanos.time_unit, ParquetTimeUnit::NANOS); - EXPECT_EQ(timestamp_nanos.extra_type_info, ParquetExtraTypeInfo::UNIT_NS); -} - -TEST(ParquetTypeTest, ResolveLogicalTimestampMatrix) { - struct Case { - ::parquet::LogicalType::TimeUnit::unit parquet_unit; - bool adjusted_to_utc; - ParquetTimeUnit expected_unit; - ParquetExtraTypeInfo expected_extra; - int expected_scale; - }; - const std::vector cases = { - {::parquet::LogicalType::TimeUnit::MILLIS, true, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MILLIS, false, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MICROS, true, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::MICROS, false, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, true, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, false, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_scale); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(test_case.adjusted_to_utc, - test_case.parquet_unit), - ::parquet::Type::INT64)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(type.doris_type), test_case.expected_scale); - EXPECT_TRUE(type.is_timestamp); - EXPECT_EQ(type.timestamp_is_adjusted_to_utc, test_case.adjusted_to_utc); - EXPECT_EQ(type.time_unit, test_case.expected_unit); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ConvertedTimeIsRejectedButConvertedTimestampIsSupported) { - const auto converted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS)); - EXPECT_EQ(converted_time.doris_type, nullptr); - EXPECT_FALSE(converted_time.supports_record_reader); - EXPECT_FALSE(converted_time.unsupported_reason.empty()); - - const auto converted_timestamp = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MILLIS)); - ASSERT_NE(converted_timestamp.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_timestamp.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(converted_timestamp.is_timestamp); - EXPECT_TRUE(converted_timestamp.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp.time_unit, ParquetTimeUnit::MILLIS); - const auto converted_timestamp_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_us", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MICROS)); - ASSERT_NE(converted_timestamp_micros.doris_type, nullptr); - EXPECT_TRUE(converted_timestamp_micros.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(converted_timestamp_micros.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(converted_timestamp_micros.doris_type), 6); - EXPECT_TRUE(converted_timestamp_micros.is_timestamp); - EXPECT_TRUE(converted_timestamp_micros.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(converted_timestamp_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); -} - -TEST(ParquetTypeTest, ResolveConvertedIntegerMappingsAndDecodedKinds) { - struct Case { - ::parquet::ConvertedType::type converted_type; - ::parquet::Type::type physical_type; - PrimitiveType expected_type; - int bit_width; - bool expected_unsigned; - DecodedValueKind expected_value_kind; - }; - const std::vector cases = { - {::parquet::ConvertedType::INT_8, ::parquet::Type::INT32, TYPE_TINYINT, 8, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_8, ::parquet::Type::INT32, TYPE_SMALLINT, 8, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_16, ::parquet::Type::INT32, TYPE_SMALLINT, 16, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_16, ::parquet::Type::INT32, TYPE_INT, 16, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_32, ::parquet::Type::INT32, TYPE_INT, 32, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_32, ::parquet::Type::INT32, TYPE_BIGINT, 32, true, - DecodedValueKind::UINT32}, - {::parquet::ConvertedType::INT_64, ::parquet::Type::INT64, TYPE_BIGINT, 64, false, - DecodedValueKind::INT64}, - {::parquet::ConvertedType::UINT_64, ::parquet::Type::INT64, TYPE_LARGEINT, 64, true, - DecodedValueKind::UINT64}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.converted_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, test_case.physical_type, - test_case.converted_type)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_value_kind); - } -} - -TEST(ParquetTypeTest, ResolveConvertedDecimalCarriers) { - struct Case { - ::parquet::Type::type physical_type; - int type_length; - int precision; - int scale; - PrimitiveType expected_type; - ParquetExtraTypeInfo expected_extra; - }; - const std::vector cases = { - {::parquet::Type::INT32, -1, 9, 2, TYPE_DECIMAL32, ParquetExtraTypeInfo::DECIMAL_INT32}, - {::parquet::Type::INT64, -1, 18, 6, TYPE_DECIMAL64, - ParquetExtraTypeInfo::DECIMAL_INT64}, - {::parquet::Type::BYTE_ARRAY, -1, 20, 5, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16, 38, 6, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20, 39, 6, TYPE_DECIMAL256, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.physical_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, test_case.physical_type, - ::parquet::ConvertedType::DECIMAL, test_case.type_length, test_case.precision, - test_case.scale)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_TRUE(type.is_decimal); - EXPECT_FALSE(type.is_string_like); - EXPECT_EQ(type.decimal_precision, test_case.precision); - EXPECT_EQ(type.decimal_scale, test_case.scale); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ResolveLogicalStringDateAndDecimalMappings) { - const std::vector> string_like_logical_types = { - ::parquet::LogicalType::String(), ::parquet::LogicalType::Enum(), - ::parquet::LogicalType::JSON(), ::parquet::LogicalType::BSON()}; - for (const auto& logical_type : string_like_logical_types) { - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::OPTIONAL, logical_type, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_STRING); - EXPECT_TRUE(type.is_string_like); - } - - const auto uuid = resolve_node(::parquet::schema::PrimitiveNode::Make( - "uuid", ::parquet::Repetition::OPTIONAL, ::parquet::LogicalType::UUID(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(uuid.doris_type, nullptr); - EXPECT_TRUE(uuid.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(uuid.doris_type), TYPE_STRING); - EXPECT_TRUE(uuid.is_string_like); - - const auto date = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Date(), - ::parquet::Type::INT32)); - ASSERT_NE(date.doris_type, nullptr); - EXPECT_EQ(primitive_type(date.doris_type), TYPE_DATEV2); - - const auto decimal64 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d64", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(18, 6), - ::parquet::Type::INT64)); - ASSERT_NE(decimal64.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal64.doris_type), TYPE_DECIMAL64); - EXPECT_TRUE(decimal64.is_decimal); - EXPECT_EQ(decimal64.decimal_precision, 18); - EXPECT_EQ(decimal64.decimal_scale, 6); - EXPECT_EQ(decimal64.extra_type_info, ParquetExtraTypeInfo::DECIMAL_INT64); - - const auto decimal128 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d128", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(38, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(decimal128.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal128.doris_type), TYPE_DECIMAL128I); - EXPECT_TRUE(decimal128.is_decimal); - EXPECT_EQ(decimal128.decimal_precision, 38); - EXPECT_EQ(decimal128.decimal_scale, 6); - EXPECT_EQ(decimal128.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d256", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(39, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - EXPECT_FALSE(decimal256.is_string_like); -} - -TEST(ParquetTypeTest, LogicalConvertedAndPhysicalFallbackLevelsAreDistinct) { - const auto logical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Int(8, true), - ::parquet::Type::INT32)); - ASSERT_NE(logical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(logical_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(logical_type.integer_bit_width, 8); - - const auto converted_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8)); - ASSERT_NE(converted_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(converted_type.integer_bit_width, 8); - - const auto physical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32)); - ASSERT_NE(physical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(physical_type.doris_type), TYPE_INT); - EXPECT_EQ(physical_type.integer_bit_width, -1); -} - -TEST(ParquetTypeTest, ResolveDecimalStringLikeFloat16AndPhysicalFallback) { - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::DECIMAL, 20, 39, 6)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_FALSE(decimal256.is_string_like); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto plain_binary = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::REQUIRED, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(plain_binary.doris_type, nullptr); - EXPECT_EQ(primitive_type(plain_binary.doris_type), TYPE_STRING); - EXPECT_TRUE(plain_binary.is_string_like); - - const auto float16 = resolve_arrow_float16_type(); - ASSERT_NE(float16.doris_type, nullptr); - EXPECT_TRUE(float16.doris_type->is_nullable()); - EXPECT_EQ(float16.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(float16.fixed_length, 2); - EXPECT_EQ(primitive_type(float16.doris_type), TYPE_FLOAT); - EXPECT_EQ(float16.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(float16.is_string_like); - EXPECT_EQ(decoded_value_kind(float16), DecodedValueKind::FIXED_BINARY); -} - -TEST(ParquetTypeTest, ResolveNullDescriptorAndPhysicalFallback) { - const auto null_type = resolve_parquet_type(nullptr); - EXPECT_EQ(null_type.doris_type, nullptr); - EXPECT_EQ(null_type.physical_type, ::parquet::Type::UNDEFINED); - EXPECT_TRUE(null_type.supports_record_reader); - - const auto int96 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT96)); - ASSERT_NE(int96.doris_type, nullptr); - EXPECT_EQ(primitive_type(int96.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(int96.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - EXPECT_EQ(decoded_value_kind(int96), DecodedValueKind::INT96); -} - -TEST(ParquetTypeTest, ResolveEveryPhysicalFallback) { - struct Case { - ::parquet::schema::NodePtr node; - PrimitiveType expected_type; - DecodedValueKind expected_kind; - bool expected_string_like = false; - }; - const std::vector cases = { - {::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BOOLEAN), - TYPE_BOOLEAN, DecodedValueKind::BOOL}, - {::parquet::schema::PrimitiveNode::Make("i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - TYPE_INT, DecodedValueKind::INT32}, - {::parquet::schema::PrimitiveNode::Make("i64", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT64), - TYPE_BIGINT, DecodedValueKind::INT64}, - {::parquet::schema::PrimitiveNode::Make("f", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FLOAT), - TYPE_FLOAT, DecodedValueKind::FLOAT}, - {::parquet::schema::PrimitiveNode::Make("d", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - TYPE_DOUBLE, DecodedValueKind::DOUBLE}, - {::parquet::schema::PrimitiveNode::Make("s", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY), - TYPE_STRING, DecodedValueKind::BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("fs", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::NONE, 4), - TYPE_STRING, DecodedValueKind::FIXED_BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("ts", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT96), - TYPE_DATETIMEV2, DecodedValueKind::INT96}, +TEST(ParquetTypeTest, DecodedValueKindUsesNativePhysicalTypes) { + ParquetTypeDescriptor descriptor; + + const std::pair cases[] = { + {tparquet::Type::BOOLEAN, DecodedValueKind::BOOL}, + {tparquet::Type::INT32, DecodedValueKind::INT32}, + {tparquet::Type::INT64, DecodedValueKind::INT64}, + {tparquet::Type::INT96, DecodedValueKind::INT96}, + {tparquet::Type::FLOAT, DecodedValueKind::FLOAT}, + {tparquet::Type::DOUBLE, DecodedValueKind::DOUBLE}, + {tparquet::Type::BYTE_ARRAY, DecodedValueKind::BINARY}, + {tparquet::Type::FIXED_LEN_BYTE_ARRAY, DecodedValueKind::FIXED_BINARY}, }; - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_type); - const auto type = resolve_node(test_case.node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_kind); - EXPECT_EQ(type.is_string_like, test_case.expected_string_like); - EXPECT_TRUE(type.supports_record_reader); + for (const auto& [physical_type, expected] : cases) { + descriptor.physical_type = physical_type; + descriptor.is_unsigned_integer = false; + descriptor.integer_bit_width = -1; + EXPECT_EQ(decoded_value_kind(descriptor), expected); } } -TEST(ParquetTypeTest, InvalidLogicalAnnotationsFallBackOrRejectAsSpecified) { - EXPECT_THROW(::parquet::LogicalType::Int(24, true), ::parquet::ParquetException); +TEST(ParquetTypeTest, DecodedValueKindPreservesUnsignedWidth) { + ParquetTypeDescriptor descriptor; + descriptor.is_unsigned_integer = true; - const auto nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(nanos_time.doris_type, nullptr); - EXPECT_EQ(primitive_type(nanos_time.doris_type), TYPE_BIGINT); - EXPECT_TRUE(nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 32; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT32); - const auto adjusted_nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns_utc", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - EXPECT_EQ(adjusted_nanos_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_nanos_time.supports_record_reader); - EXPECT_FALSE(adjusted_nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT64; + descriptor.integer_bit_width = 64; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT64); - EXPECT_THROW(::parquet::schema::PrimitiveNode::Make("f16_bad", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Float16(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 4), - ::parquet::ParquetException); + // Narrow unsigned logical integers still use the signed physical decoder; conversion + // happens after decoding so the on-disk bit width remains the single source of truth. + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 16; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::INT32); } } // namespace doris::format::parquet diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index 28c396371e4559..f5bd70292e51a3 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -17,21 +17,35 @@ #include "format_v2/table/hudi_reader.h" +#include +#include #include +#include +#include +#include +#include #include #include #include +#include #include #include +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/field.h" +#include "exec/scan/file_scanner_v2.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "format_v2/column_data.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" namespace doris::format { namespace { @@ -69,6 +83,39 @@ ColumnDefinition make_file_column(int32_t id, const std::string& name, const Dat return field; } +ColumnDefinition make_table_column(int32_t id, const std::string& name, const DataTypePtr& type) { + ColumnDefinition column; + column.identifier = Field::create_field(id); + column.name = name; + column.type = make_nullable(type); + return column; +} + +Block build_table_block(const std::vector& columns) { + Block block; + for (const auto& column : columns) { + block.insert({column.type->create_column(), column.type, column.name}); + } + return block; +} + +void write_int_parquet_file(const std::string& file_path, const std::vector& values) { + arrow::Int32Builder value_builder; + for (const auto value : values) { + ASSERT_TRUE(value_builder.Append(value).ok()); + } + std::shared_ptr value_array; + ASSERT_TRUE(value_builder.Finish(&value_array).ok()); + const auto table = arrow::Table::Make( + arrow::schema({arrow::field("id", arrow::int32(), false)}), {value_array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr output = *file_result; + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, + static_cast(values.size()))); +} + TTableFormatFileDesc hudi_table_format_desc(std::optional schema_id) { TTableFormatFileDesc table_format_params; table_format_params.__set_table_format_type("hudi"); @@ -80,6 +127,71 @@ TTableFormatFileDesc hudi_table_format_desc(std::optional schema_id) { return table_format_params; } +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + +class AppendTrackingTableReader final : public TableReader { +public: + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { + appended_conjuncts += conjuncts.size(); + owned_conjuncts += _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + return Status::OK(); + } + + size_t appended_conjuncts = 0; + size_t owned_conjuncts = 0; +}; + +class OneRowTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status get_block(Block* block, bool* eos) override { + auto column = ColumnInt32::create(); + column->insert_value(1); + block->replace_by_position(0, std::move(column)); + *eos = false; + return Status::OK(); + } +}; + +class StatefulHybridPredicate final : public VExpr { +public: + explicit StatefulHybridPredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + _observed_invocations->push_back(_invocation++); + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_constant() const override { return false; } + bool is_deterministic() const override { return false; } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulHybridPredicate"; +}; + // Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical // file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id // mapping, including renamed nested children. @@ -201,5 +313,207 @@ TEST(HudiHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 123); } +TEST(HudiHybridReaderTest, ReportsActiveChildMaterializedBlockStats) { + hudi::HudiHybridReader reader; + reader.TEST_install_batch_size_children(); + reader._current_split_reader = reader._native_reader.get(); + reader._native_reader->_last_materialized_block_stats = { + .has_materialized_input = true, .rows = 7, .bytes = 70, .allocated_bytes = 96}; + + const auto& stats = reader.last_materialized_block_stats(); + EXPECT_TRUE(stats.has_materialized_input); + EXPECT_EQ(stats.rows, 7); + EXPECT_EQ(stats.bytes, 70); + EXPECT_EQ(stats.allocated_bytes, 96); +} + +TEST(HudiHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + auto native_reader = std::make_unique(); + auto jni_reader = std::make_unique(); + auto* native_reader_ptr = native_reader.get(); + auto* jni_reader_ptr = jni_reader.get(); + reader._native_reader = std::move(native_reader); + reader._jni_reader = std::move(jni_reader); + + auto literal = VLiteral::create_shared(std::make_shared(), + Field::create_field(1)); + ASSERT_TRUE(reader.append_conjuncts_with_ownership( + {VExprContext::create_shared(std::move(literal))}, 0) + .ok()); + EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); +} + +TEST(HudiHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("hudi_scanner_stateful_residual"); + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + auto hybrid_reader = std::make_unique(); + auto* hybrid_reader_ptr = hybrid_reader.get(); + hybrid_reader_ptr->TEST_set_child_reader_factories( + [] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + std::vector observed_invocations; + auto conjunct = VExprContext::create_shared( + std::make_shared(&observed_invocations)); + ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); + scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + ASSERT_TRUE(hybrid_reader_ptr + ->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto run_split = [&](FileFormat format, TFileFormatType::type thrift_format) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range.__set_format_type(thrift_format); + ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); + ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); + }; + run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + run_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); + run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + + EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); +} + +TEST(HudiHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { + hudi::HudiHybridReader reader; + reader.TEST_install_batch_size_children(); + reader.TEST_set_child_condition_cache_hits(2, 7); + EXPECT_EQ(reader.condition_cache_hit_count(), 9); +} + +TEST(HudiHybridReaderTest, NativeCountStarReportsMetadataRowsThroughHybridReader) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_hudi_hybrid_count_star_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "base-file.parquet").string(); + write_int_parquet_file(file_path, {1, 2, 3}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + io_ctx->file_cache_stats = &file_cache_stats; + ShardedKVCache cache(1); + + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.cache = &cache; + split_options.current_split_format = FileFormat::PARQUET; + split_options.current_range.__set_path(file_path); + split_options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(file_path))); + split_options.current_range.__set_format_type(TFileFormatType::FORMAT_PARQUET); + split_options.current_range.__set_table_format_params(hudi_table_format_desc(std::nullopt)); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 3); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(HudiHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileFormatType::type thrift_format) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range.__set_format_type(thrift_format); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + verify_first_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp new file mode 100644 index 00000000000000..a219ba37f3dd8d --- /dev/null +++ b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" + +#include + +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::iceberg { +namespace { + +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_delete_files({delete_file}); + TTableFormatFileDesc table_format_desc; + table_format_desc.__set_iceberg_params(std::move(iceberg_desc)); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_desc)); + return range; +} + +TEST(IcebergPositionDeleteSysTableV2ProfileTest, UsesDistinctProfileForNestedPositionReader) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("position_delete_system_table_profile"); + TFileScanRangeParams params; + std::vector file_slot_descs; + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + delete_file.__set_path("/not-opened-during-prepare.parquet"); + + IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._scan_params = ¶ms; + reader._file_slot_descs = &file_slot_descs; + reader._current_range = range_with_delete_file(delete_file); + ASSERT_TRUE(reader._init_split().ok()); + + ASSERT_NE(reader._position_reader_profile, nullptr); + EXPECT_NE(reader._position_reader_profile, &profile); + EXPECT_EQ(profile.get_child("IcebergPositionDeleteFileReader"), + reader._position_reader_profile); +} + +} // namespace +} // namespace doris::format::iceberg diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 0831c98750ecbf..c42d68933f2014 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -21,15 +21,19 @@ #include #include #include +#include #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -55,6 +59,7 @@ #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" #include "exec/common/endian.h" +#include "exec/scan/access_path_parser.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" @@ -72,6 +77,7 @@ #include "runtime/runtime_state.h" #include "storage/segment/condition_cache.h" #include "util/debug_points.h" +#include "util/hash_util.hpp" namespace doris::format { namespace { @@ -229,7 +235,9 @@ class IcebergTableReaderScanRequestTestHelper final class IcebergTableReaderMappingModeTestHelper final : public doris::format::iceberg::IcebergTableReader { public: - TableColumnMappingMode mapping_mode_for_schema(std::vector file_schema) { + TableColumnMappingMode mapping_mode_for_schema(std::vector file_schema, + TFileScanRangeParams* scan_params = nullptr) { + _scan_params = scan_params; _data_reader.file_schema = std::move(file_schema); return mapping_mode(); } @@ -462,6 +470,39 @@ void write_single_int_parquet_file(const std::string& file_path, const std::stri builder.build())); } +void write_unsupported_time_first_int_parquet_file(const std::string& file_path, + const std::vector& ids) { + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + // Arrow writes time32 as isAdjustedToUTC=false, which Doris supports. Use Parquet's low-level + // writer so the first physical column is the deliberately unsupported adjusted TIME_MILLIS + // shape from the review report. The query will project only the following supported `id`. + const auto unsupported_time = ::parquet::schema::PrimitiveNode::Make( + "unsupported_time", ::parquet::Repetition::REQUIRED, + ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), + ::parquet::Type::INT32); + const auto id = ::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32); + const auto schema_node = ::parquet::schema::GroupNode::Make( + "schema", ::parquet::Repetition::REQUIRED, {unsupported_time, id}); + const auto schema = std::static_pointer_cast<::parquet::schema::GroupNode>(schema_node); + + auto writer = ::parquet::ParquetFileWriter::Open(out, schema); + auto* row_group = writer->AppendRowGroup(); + auto* time_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + std::vector times(ids.size(), 1000); + const auto row_count = cast_set(ids.size()); + EXPECT_EQ(time_writer->WriteBatch(row_count, nullptr, nullptr, times.data()), row_count); + time_writer->Close(); + auto* id_writer = static_cast<::parquet::Int32Writer*>(row_group->NextColumn()); + EXPECT_EQ(id_writer->WriteBatch(row_count, nullptr, nullptr, ids.data()), row_count); + id_writer->Close(); + row_group->Close(); + writer->Close(); +} + void write_two_int_parquet_file(const std::string& file_path, const std::string& first_name, const std::vector& first_values, std::optional first_field_id, @@ -495,6 +536,77 @@ void write_two_int_parquet_file(const std::string& file_path, const std::string& builder.build())); } +void write_recursive_idless_wrapper_parquet_file(const std::string& file_path, int32_t value, + bool outer_has_field_id = true) { + const auto leaf_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"30"}); + auto leaf_field = arrow::field("leaf", arrow::int32(), false)->WithMetadata(leaf_metadata); + auto inner_result = arrow::StructArray::Make({build_int32_array({value})}, {leaf_field}); + ASSERT_TRUE(inner_result.ok()) << inner_result.status(); + auto inner_field = arrow::field("inner", arrow::struct_({leaf_field}), false); + auto outer_result = arrow::StructArray::Make({*inner_result}, {inner_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + auto outer_field = arrow::field("outer", arrow::struct_({inner_field}), false); + if (outer_has_field_id) { + outer_field = + outer_field->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + } + auto table = arrow::Table::Make(arrow::schema({outer_field}), {*outer_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_nullable_idless_struct_with_sibling_id_parquet_file(const std::string& file_path) { + const auto sibling_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"2"}); + auto sibling_field = arrow::field("b", arrow::int32(), false)->WithMetadata(sibling_metadata); + const auto null_bitmap = arrow::Buffer::FromString(std::string("\x02", 1)); + auto struct_result = + arrow::StructArray::Make({build_int32_array({0, 42})}, {sibling_field}, null_bitmap, 1); + ASSERT_TRUE(struct_result.ok()) << struct_result.status(); + auto struct_field = arrow::field("s", arrow::struct_({sibling_field}), true); + auto table = arrow::Table::Make(arrow::schema({struct_field}), {*struct_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + +void write_nullable_renamed_struct_child_parquet_file(const std::string& file_path) { + const auto child_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"1"}); + auto child_field = arrow::field("b", arrow::int32(), false)->WithMetadata(child_metadata); + const auto null_bitmap = arrow::Buffer::FromString(std::string("\x02", 1)); + auto struct_result = + arrow::StructArray::Make({build_int32_array({0, 10})}, {child_field}, null_bitmap, 1); + ASSERT_TRUE(struct_result.ok()) << struct_result.status(); + auto struct_field = + arrow::field("s", arrow::struct_({child_field}), true) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"10"})); + auto table = arrow::Table::Make(arrow::schema({struct_field}), {*struct_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + void write_timestamp_int_parquet_file(const std::string& file_path, const std::vector& timestamps, const std::vector& ids) { @@ -679,7 +791,8 @@ int64_t write_iceberg_deletion_vector_file(const std::string& file_path, BigEndian::Store32(blob.data(), total_length); constexpr char DV_MAGIC[] = {'\xD1', '\xD3', '\x39', '\x64'}; memcpy(blob.data() + 4, DV_MAGIC, 4); - BigEndian::Store32(blob.data() + 8 + bitmap_size, 0); + const uint32_t crc = HashUtil::zlib_crc_hash(blob.data() + 4, total_length, 0); + BigEndian::Store32(blob.data() + 8 + bitmap_size, crc); std::ofstream output(file_path, std::ios::binary); EXPECT_TRUE(output.is_open()); @@ -696,6 +809,12 @@ class ScopedDebugPoint { DebugPoints::instance()->add(_name); } + ScopedDebugPoint(std::string name, std::function callback) + : _name(std::move(name)), _enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add_with_callback(_name, std::move(callback)); + } + ~ScopedDebugPoint() { DebugPoints::instance()->remove(_name); config::enable_debug_points = _enable_debug_points; @@ -885,9 +1004,19 @@ TFileScanRangeParams make_local_parquet_scan_params() { TFileScanRangeParams scan_params; scan_params.__set_file_type(TFileType::FILE_LOCAL); scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); return scan_params; } +TColumnAccessPath nested_data_access_path(std::vector path) { + TColumnAccessPath access_path; + access_path.__set_type(TAccessPathType::DATA); + TDataAccessPath data_path; + data_path.__set_path(std::move(path)); + access_path.__set_data_access_path(std::move(data_path)); + return access_path; +} + std::shared_ptr make_io_context(io::FileReaderStats* file_reader_stats, io::FileCacheStatistics* file_cache_stats) { auto io_ctx = std::make_shared(); @@ -1017,11 +1146,6 @@ VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { return ctx; } -void apply_final_conjuncts(Block* block, const VExprContextSPtrs& conjuncts) { - const auto status = VExprContext::filter_block(conjuncts, block, block->columns()); - ASSERT_TRUE(status.ok()) << status; -} - TEST(IcebergV2ReaderTest, IcebergVirtualColumnsUseRowLineageMetadata) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_virtual_columns_test"; @@ -1260,9 +1384,6 @@ TEST(IcebergV2ReaderTest, IcebergRowIdPredicateFiltersAfterRowLineageMaterializa bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 3); - - apply_final_conjuncts(&block, conjuncts); ASSERT_EQ(block.rows(), 1); expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); @@ -1314,9 +1435,6 @@ TEST(IcebergV2ReaderTest, IcebergLastUpdatedSequencePredicateFiltersAfterMateria bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 3); - - apply_final_conjuncts(&block, conjuncts); ASSERT_EQ(block.rows(), 1); expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); @@ -1573,8 +1691,25 @@ TEST(IcebergV2ReaderTest, IcebergDeletionVectorRejectsMissingRange) { auto status = reader.parse_deletion_vector_file(table_format_desc, &desc, &has_delete_file); EXPECT_FALSE(status.ok()); - EXPECT_TRUE(status.is()); - EXPECT_NE(status.to_string().find("missing content offset or length"), std::string::npos); + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("descriptor misses"), std::string::npos); + EXPECT_FALSE(has_delete_file); +} + +TEST(IcebergV2ReaderTest, IcebergDeletionVectorRejectsInvalidRange) { + TTableFormatFileDesc table_format_desc; + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_format_version(2); + iceberg_desc.__set_delete_files({make_iceberg_deletion_vector("dv.bin", -1, 12)}); + table_format_desc.__set_iceberg_params(iceberg_desc); + + IcebergTableReaderDeleteFileTestHelper reader; + DeleteFileDesc desc; + bool has_delete_file = false; + auto status = reader.parse_deletion_vector_file(table_format_desc, &desc, &has_delete_file); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("offset must be non-negative"), std::string::npos); EXPECT_FALSE(has_delete_file); } @@ -1843,10 +1978,12 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeIgnoresGlobalRowIdVirtualColumn) { TableColumnMappingMode::BY_FIELD_ID); } -// Covers the fallback side of the previous case. Only synthesized columns are ignored; a real data -// column without an Iceberg field id still disables field-id mapping. -TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { +// Iceberg treats a schema as ID-bearing when any physical data field has an ID. Keeping ID mode +// preserves authoritative matches even when a sibling was written without an ID. +TEST(IcebergV2ReaderTest, IcebergMappingModeUsesAnyDataColumnFieldId) { IcebergTableReaderMappingModeTestHelper reader; + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); std::vector file_schema { make_file_column(1, "id", std::make_shared()), make_file_column(2, "name", std::make_shared()), @@ -1854,7 +1991,35 @@ TEST(IcebergV2ReaderTest, IcebergMappingModeRequiresFieldIdsForDataColumns) { }; file_schema[1].identifier = Field {}; - EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema)), + EXPECT_EQ(reader.mapping_mode_for_schema(file_schema, &scan_params), + TableColumnMappingMode::BY_FIELD_ID); + + file_schema[0].identifier = Field {}; + file_schema[0].children.emplace_back( + make_file_column(3, "nested", std::make_shared())); + EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema), &scan_params), + TableColumnMappingMode::BY_FIELD_ID); +} + +TEST(IcebergV2ReaderTest, IcebergLegacyPlanKeepsAllFieldIdsMappingRule) { + IcebergTableReaderMappingModeTestHelper reader; + TFileScanRangeParams old_fe_scan_params; + std::vector file_schema { + make_file_column(1, "a", std::make_shared()), + make_file_column(2, "b", std::make_shared()), + }; + file_schema[1].identifier = Field {}; + + EXPECT_EQ(reader.mapping_mode_for_schema(std::move(file_schema), &old_fe_scan_params), + TableColumnMappingMode::BY_NAME); + + auto nested = make_file_column(10, "s", std::make_shared()); + nested.children = { + make_file_column(1, "a", std::make_shared()), + make_file_column(2, "b", std::make_shared()), + }; + nested.children[1].identifier = Field {}; + EXPECT_EQ(reader.mapping_mode_for_schema({std::move(nested)}, &old_fe_scan_params), TableColumnMappingMode::BY_NAME); } @@ -1938,6 +2103,9 @@ TEST(IcebergV2ReaderTest, IcebergTableLevelCountUsesAssignedRowCountWithPosition .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + // An explicit empty argument list is the new FE marker for + // COUNT(*)/COUNT(1); nullopt intentionally exercises fallback. + .push_down_count_columns = std::vector {}, }) .ok()); @@ -2147,6 +2315,45 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesNullForMissingDataColumn) std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMissingKeyDoesNotReadUnsupportedUnprojectedCarrier) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_virtual_carrier_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_unsupported_time_first_int_parquet_file(file_path, {1, 2, 3}); + write_iceberg_equality_delete_parquet_file(delete_file_path, 1, 7, "added_column"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + // The missing data key is NULL, so it does not match delete key 7. More importantly, the + // hidden row-count dependency must use virtual row position instead of the first physical + // TIME_MILLIS column, which is unsupported and was not requested by this `id` projection. + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 2, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesInitialDefaultForMissingDataColumn) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_missing_default_test"; @@ -2240,7 +2447,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); - const std::string binary_default("\x00\x01\x02\xff", 4); + static_assert(sizeof("0123456789abcdef0123456789abcdef") - 1 > StringView::kInlineSize); + const std::string binary_default = "0123456789abcdef0123456789abcdef"; write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, "added_binary"); @@ -2249,10 +2457,12 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultFor auto scan_params = make_local_parquet_scan_params(); scan_params.__set_current_schema_id(100); scan_params.__set_history_schema_info({external_schema( - 100, - {external_schema_field("id", 0), - external_schema_field("added_binary", 1, {}, "AAEC/w==", - external_primitive_type(TPrimitiveType::VARBINARY, 4), true)})}); + 100, {external_schema_field("id", 0), + external_schema_field( + "added_binary", 1, {}, "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + external_primitive_type(TPrimitiveType::VARBINARY, + static_cast(binary_default.size())), + true)})}); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -2443,7 +2653,401 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesNameMappingWithoutFileFieldId std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { +TEST(IcebergV2ReaderTest, ParquetRecursivelyRetainsIdlessWrapperForSelectedLeafId) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_recursive_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ParquetReadsIdlessWrapperWithAuthoritativeEmptyMapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_authoritative_empty_idless_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_recursive_idless_wrapper_parquet_file(file_path, 42, false); + + const auto int_type = std::make_shared(); + auto leaf = make_table_column(30, "leaf", int_type); + auto inner_type = std::make_shared(DataTypes {int_type}, Strings {"leaf"}); + auto inner = make_table_column(20, "inner", inner_type); + inner.children = {leaf}; + inner.has_name_mapping = true; + auto outer_type = std::make_shared(DataTypes {inner_type}, Strings {"inner"}); + auto outer = make_table_column(10, "outer", outer_type); + outer.children = {inner}; + outer.has_name_mapping = true; + std::vector projected_columns = {outer}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto& outer_result = + assert_cast(expect_not_null_table_column(block, 0)); + const auto& inner_result = assert_cast( + expect_not_null_nullable_nested_column(outer_result.get_column(0))); + const auto& leaf_result = assert_cast( + expect_not_null_nullable_nested_column(inner_result.get_column(0))); + ASSERT_EQ(leaf_result.size(), 1); + EXPECT_EQ(leaf_result.get_element(0), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ParquetUsesUnprojectedSiblingIdToRetainNullableWrapper) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_unprojected_sibling_id_wrapper_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_idless_struct_with_sibling_id_parquet_file(file_path); + + const auto int_type = std::make_shared(); + auto projected_a = make_table_column(1, "a", int_type); + projected_a.initial_default_value = "7"; + auto projected_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto projected_s = make_table_column(10, "s", projected_struct_type); + projected_s.children = {projected_a}; + std::vector projected_columns = {projected_s}; + + auto schema_a = + external_schema_field("a", 1, {}, "7", external_primitive_type(TPrimitiveType::INT)); + auto schema_b = external_schema_field("b", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + schema::external::TStructField struct_children; + struct_children.__set_fields({schema_a, schema_b}); + auto schema_s = external_schema_field("s", 10, {}, std::nullopt, + external_primitive_type(TPrimitiveType::STRUCT)); + schema_s.field_ptr->nestedField.__set_struct_field(std::move(struct_children)); + schema_s.field_ptr->__isset.nestedField = true; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {schema_s})}); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_a = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_a.is_null_at(1)); + const auto& values = assert_cast(nullable_a.get_nested_column()); + EXPECT_EQ(values.get_element(1), 7); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ReusedRootNameReadsNewFieldInitialDefault) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_reused_root_name_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_single_int_parquet_file(file_path, "b", {10}, 1); + + auto renamed_b = external_schema_field("renamed_b", 1, {"b"}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + renamed_b.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_b = + external_schema_field("b", 2, {}, "7", external_primitive_type(TPrimitiveType::INT)); + current_b.field_ptr->__set_name_mapping({}); + current_b.field_ptr->__set_name_mapping_is_authoritative(true); + + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {renamed_b, current_b})}); + + auto projected_b = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &projected_b).ok()); + std::vector projected_columns = {projected_b}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({7})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, LegacyPlanRetainsOrderedRootNameAndAliasLookupWithAllFieldIds) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_legacy_reused_root_name_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_two_int_parquet_file(file_path, "renamed_b", {10}, 1, "b", {20}, 2); + + auto renamed_b = external_schema_field("renamed_b", 1, {"b"}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + renamed_b.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_b = external_schema_field("b", 2, {}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + current_b.field_ptr->__set_name_mapping({}); + current_b.field_ptr->__set_name_mapping_is_authoritative(true); + + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_file_type(TFileType::FILE_LOCAL); + old_fe_scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + old_fe_scan_params.__set_current_schema_id(100); + old_fe_scan_params.__set_history_schema_info({external_schema(100, {renamed_b, current_b})}); + + auto projected_b = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &old_fe_scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &projected_b).ok()); + ASSERT_EQ(projected_b.get_identifier_field_id(), 1); + std::vector projected_columns = {projected_b}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &old_fe_scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({10})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, ReusedNestedNameReadsNewFieldInitialDefault) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_reused_nested_name_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_renamed_struct_child_parquet_file(file_path); + + auto renamed_b = external_schema_field("renamed_b", 1, {"b"}, std::nullopt, + external_primitive_type(TPrimitiveType::INT)); + renamed_b.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_b = + external_schema_field("b", 2, {}, "7", external_primitive_type(TPrimitiveType::INT)); + current_b.field_ptr->__set_name_mapping({}); + current_b.field_ptr->__set_name_mapping_is_authoritative(true); + schema::external::TStructField struct_children; + struct_children.__set_fields({renamed_b, current_b}); + auto schema_s = external_schema_field("s", 10, {}, std::nullopt, + external_primitive_type(TPrimitiveType::STRUCT)); + schema_s.field_ptr->nestedField.__set_struct_field(std::move(struct_children)); + schema_s.field_ptr->__isset.nestedField = true; + + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {schema_s})}); + + const auto int_type = std::make_shared(); + auto struct_type = std::make_shared(DataTypes {int_type}, Strings {"b"}); + auto projected_s = make_table_column(-1, "s", struct_type); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &projected_s).ok()); + ASSERT_TRUE(context.schema_column.has_value()); + ASSERT_TRUE(AccessPathParser::build_nested_children( + &projected_s, + std::vector {nested_data_access_path({"s", "b"})}, + &*context.schema_column) + .ok()); + std::vector projected_columns = {projected_s}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_b = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_b.is_null_at(1)); + const auto& values = assert_cast(nullable_b.get_nested_column()); + EXPECT_EQ(values.get_element(1), 7); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, LegacyPlanUsesNestedNameMappingForMixedFieldIds) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_legacy_nested_name_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_nullable_idless_struct_with_sibling_id_parquet_file(file_path); + + const auto int_type = std::make_shared(); + auto projected_a = make_table_column(1, "a", int_type); + projected_a.name_mapping = {"b"}; + projected_a.has_name_mapping = true; + auto projected_struct_type = + std::make_shared(DataTypes {int_type}, Strings {"a"}); + auto projected_s = make_table_column(10, "s", projected_struct_type); + projected_s.children = {projected_a}; + std::vector projected_columns = {projected_s}; + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_file_type(TFileType::FILE_LOCAL); + old_fe_scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &old_fe_scan_params, io_ctx, &state, &profile); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + const auto result = block.get_by_position(0).column->convert_to_full_column_if_const(); + const auto& nullable_s = assert_cast(*result); + ASSERT_EQ(nullable_s.size(), 2); + EXPECT_TRUE(nullable_s.is_null_at(0)); + EXPECT_FALSE(nullable_s.is_null_at(1)); + const auto& struct_s = assert_cast(nullable_s.get_nested_column()); + const auto& nullable_a = assert_cast(struct_s.get_column(0)); + EXPECT_FALSE(nullable_a.is_null_at(1)); + const auto& values = assert_cast(nullable_a.get_nested_column()); + EXPECT_EQ(values.get_element(1), 42); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeletePrefersExistingFieldIdInMixedSchema) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_equality_delete_stale_field_id_test"; std::filesystem::remove_all(test_dir); @@ -2451,8 +3055,8 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { const auto file_path = (test_dir / "split.parquet").string(); const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); - // The real key has no id and is mapped by its historical name. A different physical column - // carries the stale id 0, forcing the entire split into BY_NAME mode. + // Iceberg's existential hasIds contract makes the physical field carrying ID 0 authoritative; + // an ID-less alias cannot switch a mixed schema back to name projection. write_two_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt, "stale_key", {100, 200, 300}, 0); write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); @@ -2477,7 +3081,7 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); - EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({100, 200, 300})); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); @@ -2839,7 +3443,25 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteFileIsReusedAcrossSplits) { first_split.cache = &cache; first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); - ASSERT_TRUE(reader.prepare_split(first_split).ok()); + const auto* total_timer = profile.get_counter("TableReader"); + const auto* prepare_timer = profile.get_counter("PrepareSplitTime"); + ASSERT_NE(total_timer, nullptr); + ASSERT_NE(prepare_timer, nullptr); + const int64_t total_before = total_timer->value(); + const int64_t prepare_before = prepare_timer->value(); + constexpr int64_t DELETE_FILE_DELAY_NS = 8'000'000; + { + ScopedDebugPoint delay_delete_file_scan( + "IcebergTableReader.prepare_split.before_delete_file_scan", + [] { std::this_thread::sleep_for(std::chrono::milliseconds(8)); }); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + } + const int64_t total_delta = total_timer->value() - total_before; + const int64_t prepare_delta = prepare_timer->value() - prepare_before; + // A cache-miss delete-file scan is derived prepare work; both common parent timers must + // contain it so the expensive miss cannot disappear between profile levels. + EXPECT_GE(prepare_delta, DELETE_FILE_DELAY_NS); + EXPECT_GE(total_delta, DELETE_FILE_DELAY_NS); EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); // The cached delete file contains entries for every referenced data file, so another split can diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 6a3f20fac52726..77466439c6be78 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -23,11 +23,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include "core/assert_cast.h" @@ -43,6 +45,9 @@ #include "core/data_type/data_type_string.h" #include "core/field.h" #include "exec/common/endian.h" +#include "exec/scan/file_scanner_v2.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "format/format_common.h" #include "format/table/deletion_vector_reader.h" #include "format/table/paimon_reader.h" @@ -60,6 +65,71 @@ namespace doris::format { namespace { +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + +class AppendTrackingTableReader final : public TableReader { +public: + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { + appended_conjuncts += conjuncts.size(); + owned_conjuncts += _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + return Status::OK(); + } + + size_t appended_conjuncts = 0; + size_t owned_conjuncts = 0; +}; + +class OneRowTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status get_block(Block* block, bool* eos) override { + auto column = ColumnInt32::create(); + column->insert_value(1); + block->replace_by_position(0, std::move(column)); + *eos = false; + return Status::OK(); + } +}; + +class StatefulHybridPredicate final : public VExpr { +public: + explicit StatefulHybridPredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + _observed_invocations->push_back(_invocation++); + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_constant() const override { return false; } + bool is_deterministic() const override { return false; } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulHybridPredicate"; +}; + DataTypePtr table_type(const DataTypePtr& type) { return type->is_nullable() ? type : make_nullable(type); } @@ -461,6 +531,20 @@ TEST(PaimonReaderTest, DeletionVectorCacheKeyIncludesOffsetAndLength) { EXPECT_NE(first_desc.key, different_length_desc.key); } +TEST(PaimonReaderTest, DeletionVectorRejectsInvalidRange) { + auto table_format_params = make_paimon_table_format_desc("dv.bin", -1, 4); + + paimon::PaimonReader reader; + DeleteFileDesc desc; + bool has_delete_file = false; + auto status = + reader.TEST_parse_deletion_vector_file(table_format_params, &desc, &has_delete_file); + + EXPECT_TRUE(status.is()); + EXPECT_NE(status.to_string().find("offset must be non-negative"), std::string::npos); + EXPECT_FALSE(has_delete_file); +} + TEST(PaimonReaderTest, DecodeDeletionVectorBufferUsesSharedFormatHelper) { // Scenario: format_v2 TableReader reads a raw Paimon BitmapDeletionVector range and delegates // the binary parsing to the same helper used by the format reader path. @@ -661,6 +745,161 @@ TEST(PaimonHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 321); } +TEST(PaimonHybridReaderTest, ReportsActiveChildMaterializedBlockStats) { + paimon::PaimonHybridReader reader; + reader.TEST_install_batch_size_children(); + reader._current_split_reader = reader._native_reader.get(); + reader._native_reader->_last_materialized_block_stats = { + .has_materialized_input = true, .rows = 7, .bytes = 70, .allocated_bytes = 96}; + + const auto& stats = reader.last_materialized_block_stats(); + EXPECT_TRUE(stats.has_materialized_input); + EXPECT_EQ(stats.rows, 7); + EXPECT_EQ(stats.bytes, 70); + EXPECT_EQ(stats.allocated_bytes, 96); +} + +TEST(PaimonHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + auto native_reader = std::make_unique(); + auto jni_reader = std::make_unique(); + auto* native_reader_ptr = native_reader.get(); + auto* jni_reader_ptr = jni_reader.get(); + reader._native_reader = std::move(native_reader); + reader._jni_reader = std::move(jni_reader); + + auto literal = VLiteral::create_shared(std::make_shared(), + Field::create_field(1)); + ASSERT_TRUE(reader.append_conjuncts_with_ownership( + {VExprContext::create_shared(std::move(literal))}, 0) + .ok()); + EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); +} + +TEST(PaimonHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("paimon_scanner_stateful_residual"); + auto scan_params = make_local_parquet_scan_params(); + auto hybrid_reader = std::make_unique(); + auto* hybrid_reader_ptr = hybrid_reader.get(); + hybrid_reader_ptr->TEST_set_child_reader_factories( + [] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + std::vector observed_invocations; + auto conjunct = VExprContext::create_shared( + std::make_shared(&observed_invocations)); + ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); + scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + ASSERT_TRUE(hybrid_reader_ptr + ->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto run_split = [&](FileFormat format, TFileRangeDesc range) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range = std::move(range); + ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); + ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); + }; + run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + run_split(FileFormat::JNI, make_paimon_jni_range()); + run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + + EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); +} + +TEST(PaimonHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { + paimon::PaimonHybridReader reader; + reader.TEST_install_batch_size_children(); + reader.TEST_set_child_condition_cache_hits(3, 5); + EXPECT_EQ(reader.condition_cache_hit_count(), 8); +} + +TEST(PaimonHybridReaderTest, NativeCountColumnReportsMetadataRowsThroughHybridReader) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_paimon_hybrid_count_column_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "data-file.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.cache = &cache; + split_options.current_split_format = FileFormat::PARQUET; + split_options.current_range = make_paimon_native_range(TFileFormatType::FORMAT_PARQUET); + split_options.current_range.__set_path(file_path); + split_options.current_range.__set_file_size( + static_cast(std::filesystem::file_size(file_path))); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 3); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) { RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; @@ -696,6 +935,49 @@ TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) { ASSERT_TRUE(reader.close().ok()); } +TEST(PaimonHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileRangeDesc range) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range = std::move(range); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, + make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + verify_first_split(FileFormat::JNI, make_paimon_jni_range()); +} + TEST(PaimonJniReaderTest, BuildScannerParamsKeepsExplicitIOManagerTempDir) { auto scan_params = make_paimon_jni_scan_params(); scan_params.__set_paimon_options({ diff --git a/be/test/format_v2/table/remote_doris_reader_test.cpp b/be/test/format_v2/table/remote_doris_reader_test.cpp index b17f82f505c2c9..a8affbde3117a3 100644 --- a/be/test/format_v2/table/remote_doris_reader_test.cpp +++ b/be/test/format_v2/table/remote_doris_reader_test.cpp @@ -18,11 +18,17 @@ #include "format_v2/table/remote_doris_reader.h" #include +#include #include #include +#include +#include +#include #include +#include #include +#include #include #include #include @@ -48,6 +54,7 @@ #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "testutil/desc_tbl_builder.h" +#include "testutil/mock/mock_runtime_state.h" namespace doris::format::remote_doris { namespace { @@ -95,6 +102,122 @@ TFileRangeDesc remote_doris_range() { return range; } +class BlockingRecordBatchReader final : public arrow::RecordBatchReader { +public: + std::shared_ptr schema() const override { + return arrow::schema({arrow::field("id", arrow::int32())}); + } + + arrow::Status ReadNext(std::shared_ptr* batch) override { + { + std::lock_guard lock(_mutex); + _entered = true; + } + _cv.notify_all(); + std::unique_lock lock(_mutex); + _cv.wait(lock, [this] { return _released; }); + *batch = nullptr; + return arrow::Status::OK(); + } + + bool wait_until_entered(std::chrono::milliseconds timeout) { + std::unique_lock lock(_mutex); + return _cv.wait_for(lock, timeout, [this] { return _entered; }); + } + + void release() { + { + std::lock_guard lock(_mutex); + _released = true; + } + _cv.notify_all(); + } + +private: + std::mutex _mutex; + std::condition_variable _cv; + bool _entered = false; + bool _released = false; +}; + +class BlockingFlightServer final : public arrow::flight::FlightServerBase { +public: + enum class Mode { DO_GET, NEXT }; + + explicit BlockingFlightServer(Mode mode) + : _mode(mode), _batch_reader(std::make_shared()) {} + + arrow::Status start() { + auto location = arrow::flight::Location::ForGrpcTcp("localhost", 0); + if (!location.ok()) { + return location.status(); + } + // FlightServerBase::Init starts serving immediately; the blocking Serve lifecycle wrapper + // is unnecessary for an in-process unit test. + return Init(arrow::flight::FlightServerOptions(*location)); + } + + ~BlockingFlightServer() override { + release(); + static_cast(Shutdown()); + } + + arrow::Status DoGet(const arrow::flight::ServerCallContext& context, + const arrow::flight::Ticket&, + std::unique_ptr* stream) override { + if (_mode == Mode::DO_GET) { + { + std::lock_guard lock(_mutex); + _entered = true; + } + _cv.notify_all(); + std::unique_lock lock(_mutex); + while (!_released && !context.is_cancelled()) { + _cv.wait_for(lock, std::chrono::milliseconds(5)); + } + if (context.is_cancelled()) { + return arrow::Status::Cancelled("client cancelled blocked DoGet"); + } + } + *stream = std::make_unique(_batch_reader); + return arrow::Status::OK(); + } + + bool wait_until_entered(std::chrono::milliseconds timeout) { + if (_mode == Mode::NEXT) { + return _batch_reader->wait_until_entered(timeout); + } + std::unique_lock lock(_mutex); + return _cv.wait_for(lock, timeout, [this] { return _entered; }); + } + + void release() { + { + std::lock_guard lock(_mutex); + _released = true; + } + _cv.notify_all(); + _batch_reader->release(); + } + +private: + Mode _mode; + std::shared_ptr _batch_reader; + std::mutex _mutex; + std::condition_variable _cv; + bool _entered = false; + bool _released = false; +}; + +TFileRangeDesc remote_doris_range(const BlockingFlightServer& server) { + auto range = remote_doris_range(); + auto& params = range.table_format_params.remote_doris_params; + params.__set_location_uri("grpc://localhost:" + std::to_string(server.port())); + arrow::flight::Ticket ticket {.ticket = "ticket"}; + params.__set_ticket(ticket.SerializeToString().ValueOrDie()); + return range; +} + std::vector remote_slots(ObjectPool* pool, DescriptorTbl** desc_tbl) { DescriptorTblBuilder builder(pool); builder.declare_tuple() << std::make_tuple(std::make_shared(), std::string("id")) @@ -196,6 +319,16 @@ std::unique_ptr create_reader( std::move(factory)); } +std::unique_ptr create_flight_reader( + RuntimeProfile* profile, const TFileRangeDesc& range, + const std::vector& slots, std::shared_ptr io_ctx) { + auto system_properties = std::make_shared(); + auto file_description = std::make_unique(); + file_description->path = "/dummyPath"; + return std::make_unique(system_properties, file_description, + std::move(io_ctx), profile, range, slots); +} + Block make_request_block(const std::vector& schema, const std::vector& local_ids) { Block block; @@ -467,4 +600,140 @@ TEST(RemoteDorisV2ReaderTest, RejectsInvalidRemoteDorisRange) { EXPECT_FALSE(reader->init(&state).ok()); } +TEST(RemoteDorisV2ReaderTest, RuntimeCancellationInterruptsBlockedFlightDoGet) { + BlockingFlightServer server(BlockingFlightServer::Mode::DO_GET); + const auto server_status = server.start(); + ASSERT_TRUE(server_status.ok()) << server_status; + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_blocked_doget_test"); + auto io_ctx = std::make_shared(); + auto reader = create_flight_reader(&profile, remote_doris_range(server), slots, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + + auto open_result = + std::async(std::launch::async, [&] { return reader->open(std::move(request)); }); + const bool entered = server.wait_until_entered(std::chrono::seconds(2)); + if (!entered && + open_result.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) { + FAIL() << "Flight DoGet failed before reaching the server: " << open_result.get(); + } + ASSERT_TRUE(entered); + state.cancel(Status::Cancelled("cancel blocked Flight DoGet")); + const bool interrupted = + open_result.wait_for(std::chrono::milliseconds(750)) == std::future_status::ready; + if (!interrupted) { + server.release(); + } + EXPECT_TRUE(interrupted); + EXPECT_FALSE(open_result.get().ok()); +} + +TEST(RemoteDorisV2ReaderTest, BlockedDoGetRetainsQueryResourcesUntilWorkerExits) { + BlockingFlightServer server(BlockingFlightServer::Mode::DO_GET); + const auto server_status = server.start(); + ASSERT_TRUE(server_status.ok()) << server_status; + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + auto state = std::make_unique(); + std::weak_ptr resource_ctx = state->get_query_ctx()->resource_ctx(); + RuntimeProfile profile("remote_doris_v2_doget_resource_context_test"); + auto reader = create_flight_reader(&profile, remote_doris_range(server), slots, + std::make_shared()); + ASSERT_TRUE(reader->init(state.get()).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + + auto open_result = + std::async(std::launch::async, [&] { return reader->open(std::move(request)); }); + ASSERT_TRUE(server.wait_until_entered(std::chrono::seconds(2))); + state->cancel(Status::Cancelled("cancel blocked Flight DoGet")); + ASSERT_EQ(open_result.wait_for(std::chrono::milliseconds(750)), std::future_status::ready); + EXPECT_FALSE(open_result.get().ok()); + + reader.reset(); + state.reset(); + // A detached DoGet still owns query-scoped Arrow objects, so it must retain the matching + // resource context until those objects are released on the worker. + EXPECT_FALSE(resource_ctx.expired()); + server.release(); + for (int retries = 0; retries < 100 && !resource_ctx.expired(); ++retries) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + EXPECT_TRUE(resource_ctx.expired()); +} + +TEST(RemoteDorisV2ReaderTest, ScannerStopInterruptsBlockedFlightNext) { + BlockingFlightServer server(BlockingFlightServer::Mode::NEXT); + const auto server_status = server.start(); + ASSERT_TRUE(server_status.ok()) << server_status; + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_blocked_next_test"); + auto io_ctx = std::make_shared(); + auto reader = create_flight_reader(&profile, remote_doris_range(server), slots, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + const auto open_status = reader->open(std::move(request)); + ASSERT_TRUE(open_status.ok()) << open_status; + + auto block = make_request_block(schema, {0}); + size_t rows = 0; + bool eof = false; + auto next_result = + std::async(std::launch::async, [&] { return reader->get_block(&block, &rows, &eof); }); + ASSERT_TRUE(server.wait_until_entered(std::chrono::seconds(2))); + io_ctx->should_stop = true; + const bool interrupted = + next_result.wait_for(std::chrono::milliseconds(750)) == std::future_status::ready; + if (!interrupted) { + server.release(); + } + EXPECT_TRUE(interrupted); + const auto next_status = next_result.get(); + EXPECT_TRUE(!next_status.ok() || (rows == 0 && eof)); + server.release(); +} + +TEST(RemoteDorisV2ReaderTest, CancellationStopsBeforeFlightNext) { + ObjectPool pool; + DescriptorTbl* desc_tbl = nullptr; + const auto slots = remote_slots(&pool, &desc_tbl); + RuntimeState state; + RuntimeProfile profile("remote_doris_v2_reader_cancel_test"); + auto close_count = std::make_shared(0); + auto io_ctx = std::make_shared(); + auto reader = create_reader(&profile, remote_doris_range(), slots, {make_batch({"id"})}, + close_count, io_ctx); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + FileScanRequestBuilder builder(request.get()); + ASSERT_TRUE(builder.add_non_predicate_column(LocalColumnId(0)).ok()); + ASSERT_TRUE(reader->open(request).ok()); + + io_ctx->should_stop = true; + auto block = make_request_block(schema, {0}); + size_t rows = 99; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 0); + EXPECT_TRUE(eof); + EXPECT_EQ(*close_count, 1); +} + } // namespace doris::format::remote_doris diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 970e4b2153557e..d92bbf2b0ac927 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -41,6 +41,7 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_struct.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -48,11 +49,13 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" +#include "format/table/iceberg_scan_semantics.h" #include "gen_cpp/Exprs_types.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" @@ -294,6 +297,37 @@ class NonDeterministicPartitionPredicate final : public VExpr { const std::string _expr_name = "NonDeterministicPartitionPredicate"; }; +class StatefulSequencePredicate final : public VExpr { +public: + explicit StatefulSequencePredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(_observed_invocations != nullptr); + _observed_invocations->push_back(_invocation++); + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_observed_invocations); + return Status::OK(); + } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulSequencePredicate"; +}; + class NullableArrayBigintDefaultExpr final : public VExpr { public: explicit NullableArrayBigintDefaultExpr(DataTypePtr data_type) @@ -327,9 +361,50 @@ class NullableArrayBigintDefaultExpr final : public VExpr { class TableReaderMaterializeTestHelper final : public TableReader { public: + using TableReader::_materialize_mapping_column; using TableReader::_materialize_map_mapping_column; }; +TEST(TableReaderTest, LastProjectionDetachesNestedMapWithoutCopyingStrings) { + auto keys = ColumnString::create(); + keys->insert_data(std::string(1UL << 20, 'k').data(), 1UL << 20); + auto values = ColumnString::create(); + values->insert_data(std::string(1UL << 20, 'v').data(), 1UL << 20); + const auto* original_value_bytes = values->get_chars().data(); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(1); + auto map = ColumnMap::create(std::move(keys), std::move(values), std::move(offsets)); + auto null_map = ColumnUInt8::create(1, 0); + ColumnPtr source = ColumnNullable::create(std::move(map), std::move(null_map)); + + const auto string_type = std::make_shared(); + const auto map_type = make_nullable(std::make_shared(string_type, string_type)); + Block block; + block.insert({source, map_type, "large_map"}); + source.reset(); + + ColumnMapping mapping; + mapping.global_index = GlobalIndex(0); + mapping.table_column_name = "large_map"; + mapping.file_column_name = "large_map"; + mapping.file_local_id = 0; + mapping.file_type = map_type; + mapping.table_type = map_type; + mapping.is_trivial = true; + mapping.projection = + VExprContext::create_shared(VSlotRef::create_shared(0, 0, -1, map_type, "large_map")); + TableReaderMaterializeTestHelper reader; + ColumnPtr detached; + ASSERT_TRUE(reader._materialize_mapping_column(mapping, &block, 1, &detached, + /*take_projection_result=*/true) + .ok()); + EXPECT_EQ(block.get_by_position(0).column->size(), 0); + const auto& detached_nullable = assert_cast(*detached); + const auto& detached_map = assert_cast(detached_nullable.get_nested_column()); + const auto& detached_values = assert_cast(detached_map.get_values()); + EXPECT_EQ(detached_values.get_chars().data(), original_value_bytes); +} + VExprSPtr table_int32_sum_expr(int left_slot_id, int left_column_id, int right_slot_id, int right_column_id) { const auto int_type = std::make_shared(); @@ -490,6 +565,23 @@ void write_parquet_file(const std::string& file_path, int32_t id, const std::str builder.build())); } +void write_single_int_parquet_file(const std::string& file_path, const std::string& column_name, + int32_t value) { + auto schema = arrow::schema({arrow::field(column_name, arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array({value})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + void write_struct_parquet_file(const std::string& file_path, int32_t id) { auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false)}); arrow::StructBuilder builder( @@ -1011,7 +1103,10 @@ struct FakeFileReaderState { bool stop_during_aggregate = false; bool stop_during_read = false; bool not_found_during_init = false; + int batch_count = 1; + int get_block_count = 0; std::shared_ptr last_request; + std::optional last_aggregate_request; std::shared_ptr condition_cache_ctx; std::shared_ptr io_ctx; }; @@ -1048,7 +1143,7 @@ class FakeFileReader final : public FileReader { RETURN_IF_ERROR(FileReader::open(std::move(request))); _state->last_request = _request; ++_state->open_count; - _returned_batch = false; + _returned_batches = 0; return Status::OK(); } @@ -1057,7 +1152,8 @@ class FakeFileReader final : public FileReader { DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); DORIS_CHECK(_request != nullptr); - if (_returned_batch) { + ++_state->get_block_count; + if (_returned_batches >= _state->batch_count) { *rows = 0; *eof = true; return Status::OK(); @@ -1104,9 +1200,9 @@ class FakeFileReader final : public FileReader { DORIS_CHECK(_state->io_ctx != nullptr); _state->io_ctx->should_stop = true; } - _returned_batch = true; + ++_returned_batches; *rows = 2; - *eof = _state->eof_with_first_batch; + *eof = _state->eof_with_first_batch && _returned_batches >= _state->batch_count; if (_state->condition_cache_ctx != nullptr && !_state->condition_cache_ctx->is_hit && _state->condition_cache_ctx->filter_result != nullptr && !_state->condition_cache_ctx->filter_result->empty()) { @@ -1127,6 +1223,7 @@ class FakeFileReader final : public FileReader { if (request.agg_type != TPushAggOp::type::COUNT) { return Status::NotSupported("fake reader only supports COUNT aggregate pushdown"); } + _state->last_aggregate_request = request; if (_state->stop_during_aggregate) { DORIS_CHECK(_state->io_ctx != nullptr); _state->io_ctx->should_stop = true; @@ -1161,7 +1258,7 @@ class FakeFileReader final : public FileReader { private: std::vector _schema; std::shared_ptr _state; - bool _returned_batch = false; + int _returned_batches = 0; }; class FakeTableReader final : public TableReader { @@ -1335,9 +1432,257 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_FALSE(predicate_executed); + EXPECT_TRUE(predicate_executed); EXPECT_FALSE(eos); + // The file was still opened, proving constant pruning did not jump over the unsafe predicate; + // the predicate is evaluated only after the resulting table row is materialized. EXPECT_EQ(fake_state->open_count, 1); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, UnsafePredicateRunsAfterTableMaterialization) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto unsafe_predicate = + std::make_shared(&predicate_executed); + unsafe_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_TRUE(predicate_executed); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ScannerOwnedUnsafePredicateIsPassedButNotExecutedByTableReader) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto unsafe_predicate = + std::make_shared(&predicate_executed); + unsafe_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .table_reader_owned_conjunct_count = 0, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_EQ(reader.TEST_conjunct_count(), 1); + EXPECT_EQ(reader.TEST_table_reader_owned_conjunct_count(), 0); + EXPECT_FALSE(predicate_executed); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ResidualExpressionStateSurvivesAcrossSplits) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + std::vector observed_invocations; + auto stateful_predicate = std::make_shared(&observed_invocations); + stateful_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, stateful_predicate)}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + for (int split_index = 0; split_index < 2; ++split_index) { + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); + } + + EXPECT_EQ(observed_invocations, std::vector({0, 1})); +} + +TEST(TableReaderTest, AllFilteredResidualReturnsAfterOneMaterializedBatch) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto fake_state = std::make_shared(); + fake_state->batch_count = 2; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + EXPECT_TRUE(predicate_executed); + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); + EXPECT_EQ(fake_state->get_block_count, 1); + EXPECT_EQ(fake_state->close_count, 0); + EXPECT_TRUE(reader.last_materialized_block_stats().has_materialized_input); + EXPECT_EQ(reader.last_materialized_block_stats().rows, 2); + EXPECT_GT(reader.last_materialized_block_stats().bytes, 0); + EXPECT_GT(reader.last_materialized_block_stats().allocated_bytes, 0); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, LateConjunctFiltersAlreadyOpenSplit) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->batch_count = 2; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_EQ(block.rows(), 2); + + bool predicate_executed = false; + auto late_predicate = std::make_shared(&predicate_executed); + late_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + ASSERT_TRUE( + reader.append_conjuncts({VExprContext::create_shared(std::move(late_predicate))}).ok()); + block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(predicate_executed); + EXPECT_EQ(block.rows(), 0); + EXPECT_EQ(fake_state->get_block_count, 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ResidualFilteringHasDedicatedProfileTimer) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("scanner"); + bool predicate_executed = false; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + EXPECT_TRUE(predicate_executed); + ASSERT_NE(profile.get_counter("ResidualFilterTime"), nullptr); + EXPECT_GT(profile.get_counter("ResidualFilterTime")->value(), 0); ASSERT_TRUE(reader.close().ok()); } @@ -1388,9 +1733,11 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { EXPECT_EQ(fake_state->open_count, 1); ASSERT_NE(fake_state->last_request, nullptr); // A slotless unsafe conjunct is an ordering barrier even though it has no TableFilter entry. - // The later predicate must stay on the scanner's row-level path instead of running inside the + // The later predicate must stay on the post-materialization path instead of running inside the // file reader before the unsafe conjunct. EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); ASSERT_TRUE(reader.close().ok()); } @@ -1510,6 +1857,7 @@ TEST(TableReaderTest, RefreshedConjunctDisablesTableLevelCount) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1552,6 +1900,7 @@ TEST(TableReaderTest, PendingRuntimeFilterDisablesTableLevelCount) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1569,6 +1918,10 @@ TEST(TableReaderTest, PendingRuntimeFilterDisablesTableLevelCount) { ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_EQ(fake_state->open_count, 1); EXPECT_EQ(block.rows(), 2); + ASSERT_NE(fake_state->last_request, nullptr); + // Aggregate pushdown is disabled while a runtime filter is pending, but COUNT(*) semantics do + // not change. The retained output slot remains a value-less placeholder during row fallback. + EXPECT_TRUE(fake_state->last_request->is_count_star_placeholder(LocalColumnId(0))); ASSERT_TRUE(reader.close().ok()); } @@ -1652,6 +2005,152 @@ TEST(TableReaderTest, SlotlessConjunctDisablesAggregatePushdown) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + // The slotless predicate cannot become a TableFilter or a file-reader conjunct, but its + // presence still prevents the fake aggregate count (3) from replacing the two physical rows. + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + // The two physical rows are then filtered at the table boundary, where slotless predicates are + // evaluated exactly even though they cannot be localized to a file column. + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); + EXPECT_TRUE(predicate_executed); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, AbortSplitClearsReaderAfterIgnorableNotFound) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->not_found_during_init = true; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("missing-fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.is()) << status; + ASSERT_TRUE(reader.abort_split().ok()); + EXPECT_EQ(fake_state->init_count, 1); + EXPECT_EQ(fake_state->close_count, 1); + + fake_state->not_found_during_init = false; + split_options.current_range.__set_path("existing-fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(fake_state->init_count, 2); + EXPECT_EQ(fake_state->close_count, 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { + const auto nullable_int_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_int_type)); + set_name_identifiers(&projected_columns); + + io::FileReaderStats file_reader_stats; + auto io_ctx = std::make_shared(); + io_ctx->file_reader_stats = &file_reader_stats; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + fake_state->io_ctx = io_ctx; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 3); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); + EXPECT_EQ(file_reader_stats.read_rows, 3); + EXPECT_EQ(fake_state->close_count, 1); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + ASSERT_TRUE(fake_state->last_request != nullptr); + EXPECT_TRUE(fake_state->last_request->count_star_placeholder_columns.empty()); + ASSERT_TRUE(fake_state->last_aggregate_request.has_value()); + ASSERT_EQ(fake_state->last_aggregate_request->columns.size(), 1); + // A primitive COUNT(col) projection must reach the file reader just like a complex one. + EXPECT_EQ(fake_state->last_aggregate_request->columns[0].projection.local_id(), 0); +} + +TEST(TableReaderTest, PushDownCountEmitsAtMostOneRuntimeBatch) { + const auto nullable_bigint_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_bigint_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_bigint_type)); + set_name_identifiers(&projected_columns); + + TQueryOptions query_options; + query_options.__set_batch_size(2); + RuntimeState state {query_options, TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 5; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, }) .ok()); @@ -1659,28 +2158,174 @@ TEST(TableReaderTest, SlotlessConjunctDisablesAggregatePushdown) { split_options.current_range.__set_path("fake-table-reader-input"); ASSERT_TRUE(reader.prepare_split(split_options).ok()); + bool eos = false; + for (const size_t expected_rows : {2, 2, 1}) { + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), expected_rows); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); + } + + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 0); + EXPECT_EQ(fake_state->open_count, 1); + EXPECT_EQ(fake_state->close_count, 1); + EXPECT_TRUE(reader.current_split_uses_metadata_count()); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, PushDownCountStarIgnoresProjectedPlaceholderColumn) { + const auto nullable_int_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "nullable_id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "nullable_id", nullable_int_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + // COUNT(*) deliberately has no explicit count columns. The + // nullable_id projection is only the planner's scan placeholder. + .push_down_count_columns = std::vector {}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 3); + ASSERT_TRUE(block.check_type_and_column().ok()) << block.dump_structure(); + ASSERT_TRUE(fake_state->last_request != nullptr); + ASSERT_EQ(fake_state->last_request->count_star_placeholder_columns.size(), 1); + EXPECT_TRUE(fake_state->last_request->is_count_star_placeholder(LocalColumnId(0))); + ASSERT_TRUE(fake_state->last_aggregate_request.has_value()); + // Passing nullable_id here would implement COUNT(nullable_id) and reproduce the external ORC + // and Parquet failures where footer row counts were reduced by null values. + EXPECT_TRUE(fake_state->last_aggregate_request->columns.empty()); +} + +TEST(TableReaderTest, PushDownCountFallsBackWhenSemanticArgumentsAreAbsent) { + const auto nullable_int_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_int_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + // Simulate an old FE: it can request COUNT pushdown but cannot + // serialize push_down_count_slot_ids. + .push_down_count_columns = std::nullopt, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 2); + // The explicit aggregate_count=3 would be returned if absence were confused with COUNT(*). + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); + EXPECT_FALSE(reader.current_split_uses_metadata_count()); +} + +TEST(TableReaderTest, PushDownCountFallsBackForMultipleArguments) { + const auto nullable_int_type = make_nullable(std::make_shared()); + const auto nullable_string_type = make_nullable(std::make_shared()); + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + file_schema.push_back(make_file_column(1, "name", nullable_string_type)); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", nullable_int_type)); + projected_columns.push_back(make_table_column(1, "name", nullable_string_type)); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->aggregate_count = 3; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0), GlobalIndex(1)}, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - // The slotless predicate cannot become a TableFilter or a file-reader conjunct, but its - // presence still prevents the fake aggregate count (3) from replacing the two physical rows. - ASSERT_NE(fake_state->last_request, nullptr); - EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_FALSE(eos); EXPECT_EQ(block.rows(), 2); - EXPECT_TRUE(predicate_executed); - ASSERT_TRUE(reader.close().ok()); + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); + EXPECT_FALSE(reader.current_split_uses_metadata_count()); } -TEST(TableReaderTest, AbortSplitClearsReaderAfterIgnorableNotFound) { +TEST(TableReaderTest, PushDownCountFallsBackForNullableToRequiredMapping) { + const auto nullable_int_type = make_nullable(std::make_shared()); std::vector file_schema; - file_schema.push_back(make_file_column(0, "id", std::make_shared())); + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); + std::vector projected_columns; projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + // make_table_column models the usual nullable external-table descriptor. Override it here to + // reproduce an evolved table contract that declares the mapped physical column required. + projected_columns[0].type = std::make_shared(); set_name_identifiers(&projected_columns); RuntimeState state {TQueryOptions(), TQueryGlobals()}; auto fake_state = std::make_shared(); - fake_state->not_found_during_init = true; + fake_state->aggregate_count = 3; FakeTableReader reader(file_schema, fake_state); ASSERT_TRUE(reader.init({ .projected_columns = projected_columns, @@ -1690,55 +2335,49 @@ TEST(TableReaderTest, AbortSplitClearsReaderAfterIgnorableNotFound) { .io_ctx = nullptr, .runtime_state = &state, .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, }) .ok()); SplitReadOptions split_options; - split_options.current_range.__set_path("missing-fake-table-reader-input"); + split_options.current_range.__set_path("fake-table-reader-input"); ASSERT_TRUE(reader.prepare_split(split_options).ok()); + Block block = build_table_block(projected_columns); bool eos = false; - const auto status = reader.get_block(&block, &eos); - ASSERT_TRUE(status.is()) << status; - ASSERT_TRUE(reader.abort_split().ok()); - EXPECT_EQ(fake_state->init_count, 1); - EXPECT_EQ(fake_state->close_count, 1); - - fake_state->not_found_during_init = false; - split_options.current_range.__set_path("existing-fake-table-reader-input"); - ASSERT_TRUE(reader.prepare_split(split_options).ok()); - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_EQ(fake_state->init_count, 2); - EXPECT_EQ(fake_state->close_count, 2); - ASSERT_TRUE(reader.close().ok()); + // The normal scan rejects the nullable physical column because it cannot satisfy the required + // table contract. Footer COUNT would bypass that validation and incorrectly return 3. + EXPECT_FALSE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); } -TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { +TEST(TableReaderTest, PushDownCountFallsBackForCastMapping) { + const auto nullable_int_type = make_nullable(std::make_shared()); + const auto nullable_bigint_type = make_nullable(std::make_shared()); std::vector file_schema; - file_schema.push_back(make_file_column(0, "id", std::make_shared())); + file_schema.push_back(make_file_column(0, "id", nullable_int_type)); std::vector projected_columns; - projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(0, "id", nullable_bigint_type)); set_name_identifiers(&projected_columns); - io::FileReaderStats file_reader_stats; - auto io_ctx = std::make_shared(); - io_ctx->file_reader_stats = &file_reader_stats; - RuntimeState state {TQueryOptions(), TQueryGlobals()}; auto fake_state = std::make_shared(); fake_state->aggregate_count = 3; - fake_state->io_ctx = io_ctx; FakeTableReader reader(file_schema, fake_state); ASSERT_TRUE(reader.init({ .projected_columns = projected_columns, .conjuncts = {}, .format = FileFormat::PARQUET, .scan_params = nullptr, - .io_ctx = io_ctx, + .io_ctx = nullptr, .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = + std::vector {GlobalIndex(0)}, }) .ok()); @@ -1750,9 +2389,10 @@ TEST(TableReaderTest, PushDownCountRecordsReaderRowsBeforeClosingReader) { bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_FALSE(eos); - EXPECT_EQ(block.rows(), 3); - EXPECT_EQ(file_reader_stats.read_rows, 3); - EXPECT_EQ(fake_state->close_count, 1); + EXPECT_EQ(block.rows(), 2); + EXPECT_TRUE(block.get_by_position(0).type->equals(*nullable_bigint_type)); + // INT->BIGINT is a non-trivial mapping, so COUNT must not skip the cast/materialization path. + EXPECT_FALSE(fake_state->last_aggregate_request.has_value()); } TEST(TableReaderTest, PushDownCountStopConvertsAggregateEndOfFileToEos) { @@ -1780,6 +2420,7 @@ TEST(TableReaderTest, PushDownCountStopConvertsAggregateEndOfFileToEos) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1820,6 +2461,7 @@ TEST(TableReaderTest, DebugStringCoversReaderStateAndEnumNames) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); @@ -1942,6 +2584,224 @@ TEST(TableReaderTest, AnnotateProjectedColumnUsesCurrentHistorySchemaForNestedTy EXPECT_EQ(context.schema_column->children[1].children[1].get_identifier_field_id(), 25); } +TEST(TableReaderTest, AnnotateProjectedColumnPrefersCurrentNameOverHistoricalAlias) { + auto renamed_field = external_schema_field("renamed_b", 1, {"b"}); + renamed_field.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_field = external_schema_field("b", 2); + current_field.field_ptr->__set_name_mapping({}); + current_field.field_ptr->__set_name_mapping_is_authoritative(true); + + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema(100, {renamed_field, current_field})}); + + ColumnDefinition projected = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &projected).ok()); + + EXPECT_EQ(projected.get_identifier_field_id(), 2); + ASSERT_TRUE(projected.has_name_mapping); + EXPECT_TRUE(projected.name_mapping.empty()); +} + +TEST(TableReaderTest, LegacyPlanRetainsOrderedCurrentNameAndAliasLookup) { + auto renamed_field = external_schema_field("renamed_b", 1, {"b"}); + renamed_field.field_ptr->__set_name_mapping_is_authoritative(true); + auto current_field = external_schema_field("b", 2); + current_field.field_ptr->__set_name_mapping({}); + current_field.field_ptr->__set_name_mapping_is_authoritative(true); + + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_current_schema_id(100); + old_fe_scan_params.__set_history_schema_info( + {external_schema(100, {renamed_field, current_field})}); + + ColumnDefinition projected = make_table_column(-1, "b", std::make_shared()); + ProjectedColumnBuildContext context {.scan_params = &old_fe_scan_params}; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &projected).ok()); + + EXPECT_EQ(projected.get_identifier_field_id(), 1); + ASSERT_TRUE(projected.has_name_mapping); + EXPECT_EQ(projected.name_mapping, std::vector({"b"})); +} + +TEST(TableReaderTest, IcebergInitialDefaultMetadataOverridesGenericBinaryDefaultExpr) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_top_level_binary_initial_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 7, "unused"); + + auto binary_field = external_schema_field("added_binary", 2); + binary_field.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + binary_field.field_ptr->__set_initial_default_value_is_base64(true); + TFileScanRangeParams scan_params; + scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_1); + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {binary_field})}); + + const auto int_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); + auto id_column = make_table_column(-1, "id", int_type); + auto binary_column = make_table_column(-1, "added_binary", varbinary_type); + binary_column.default_expr = VExprContext::create_shared(VLiteral::create_shared( + binary_column.type, + Field::create_field(StringView("Ej5FZ+ibEtOkVkJmFBdAAA==")))); + ProjectedColumnBuildContext context {.scan_params = &scan_params}; + TFileScanSlotInfo slot_info; + TableReader annotation_reader; + ASSERT_TRUE( + annotation_reader.annotate_projected_column(slot_info, &context, &binary_column).ok()); + ASSERT_TRUE(binary_column.initial_default_value.has_value()); + ASSERT_TRUE(binary_column.initial_default_value_is_base64); + + std::vector projected_columns = {id_column, binary_column}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({.projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr}) + .ok()); + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.get_by_position(1).column->get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, IcebergLegacyPlanKeepsGenericBinaryDefaultExpr) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_legacy_binary_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto file_path = (test_dir / "split.parquet").string(); + write_parquet_file(file_path, 7, "unused"); + + auto binary_field = external_schema_field("added_binary", 2); + binary_field.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + binary_field.field_ptr->__set_initial_default_value_is_base64(true); + TFileScanRangeParams old_fe_scan_params; + old_fe_scan_params.__set_current_schema_id(1); + old_fe_scan_params.__set_history_schema_info({external_schema(1, {binary_field})}); + + const auto varbinary_type = std::make_shared(16); + auto binary_column = make_table_column(-1, "added_binary", varbinary_type); + binary_column.default_expr = VExprContext::create_shared(VLiteral::create_shared( + binary_column.type, + Field::create_field(StringView("Ej5FZ+ibEtOkVkJmFBdAAA==")))); + ProjectedColumnBuildContext context {.scan_params = &old_fe_scan_params}; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &binary_column).ok()); + + EXPECT_FALSE(binary_column.initial_default_value.has_value()); + ASSERT_TRUE(context.schema_column.has_value()); + EXPECT_FALSE(context.schema_column->initial_default_value.has_value()); + + auto nested_child = external_schema_field("added_binary", 2); + nested_child.field_ptr->__set_initial_default_value("Ej5FZ+ibEtOkVkJmFBdAAA=="); + nested_child.field_ptr->__set_initial_default_value_is_base64(true); + auto nested_field = external_struct_field("s", 10, {nested_child}); + TFileScanRangeParams old_fe_nested_params; + old_fe_nested_params.__set_current_schema_id(1); + old_fe_nested_params.__set_history_schema_info({external_schema(1, {nested_field})}); + auto struct_type = + std::make_shared(DataTypes {varbinary_type}, Strings {"added_binary"}); + auto struct_column = make_table_column(-1, "s", struct_type); + ProjectedColumnBuildContext nested_context {.scan_params = &old_fe_nested_params}; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &nested_context, &struct_column).ok()); + ASSERT_TRUE(nested_context.schema_column.has_value()); + ASSERT_EQ(nested_context.schema_column->children.size(), 1); + EXPECT_FALSE(nested_context.schema_column->children[0].initial_default_value.has_value()); + + auto id_column = make_table_column(-1, "id", std::make_shared()); + std::vector projected_columns = {id_column, binary_column}; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader data_reader; + ASSERT_TRUE(data_reader + .init({.projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &old_fe_scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr}) + .ok()); + ASSERT_TRUE(data_reader.prepare_split(build_split_options(file_path)).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(data_reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + EXPECT_EQ(block.get_by_position(1).column->get_data_at(0).to_string(), + "Ej5FZ+ibEtOkVkJmFBdAAA=="); + ASSERT_TRUE(data_reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(TableReaderTest, ExplicitEmptyNameMappingDoesNotMatchCurrentFileName) { + auto unmapped_field = external_schema_field("b", 2); + unmapped_field.field_ptr->__set_name_mapping({}); + unmapped_field.field_ptr->__set_name_mapping_is_authoritative(true); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {unmapped_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_TRUE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + EXPECT_FALSE(mapper.mappings()[0].file_local_id.has_value()); +} + +TEST(TableReaderTest, LegacyFeEmptyNameMappingStillMatchesCurrentFileName) { + auto legacy_field = external_schema_field("b", 2); + legacy_field.field_ptr->__set_name_mapping({}); + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(1); + scan_params.__set_history_schema_info({external_schema(1, {legacy_field})}); + + const auto int_type = std::make_shared(); + ColumnDefinition table_column = make_table_column(-1, "b", int_type); + ProjectedColumnBuildContext context; + context.scan_params = &scan_params; + TFileScanSlotInfo slot_info; + TableReader reader; + ASSERT_TRUE(reader.annotate_projected_column(slot_info, &context, &table_column).ok()); + ASSERT_FALSE(table_column.has_name_mapping); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE( + mapper.create_mapping({table_column}, {}, {make_file_column(0, "b", int_type)}).ok()); + ASSERT_EQ(mapper.mappings().size(), 1); + ASSERT_TRUE(mapper.mappings()[0].file_local_id.has_value()); + EXPECT_EQ(*mapper.mappings()[0].file_local_id, 0); +} + TEST(TableReaderTest, ComplexRematerializeCastsScalarChildToTableType) { const auto string_type = std::make_shared(); const auto nullable_string_type = make_nullable(string_type); @@ -2146,9 +3006,10 @@ TEST(TableReaderTest, ConditionCacheSkipsRequestWithoutFileLocalConjuncts) { ASSERT_TRUE(reader.close().ok()); } -// Scenario: runtime filters can arrive late and are not represented by the stable predicate digest. -// A MISS must not insert a bitmap for `stable predicate AND runtime filter` under the stable digest. -TEST(TableReaderTest, ConditionCacheSkipsRuntimeFilterConjunct) { +// Scenario: a standalone caller has only the initial digest for stable predicate P, while its +// current conjunct snapshot also contains an RF. Without an explicit split digest, TableReader must +// not store P AND RF under P's stale key. +TEST(TableReaderTest, ConditionCacheSkipsRuntimeFilterWithoutSplitDigest) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); @@ -2186,6 +3047,60 @@ TEST(TableReaderTest, ConditionCacheSkipsRuntimeFilterConjunct) { ASSERT_TRUE(reader.close().ok()); } +// Scenario: FileScannerV2 supplies a non-zero digest computed from the exact Ready RF payload in +// this split. The RF wrapper is no longer a reason to disable condition cache; a MISS context is +// created and can be published under that payload-specific key after EOF. +TEST(TableReaderTest, ConditionCacheAllowsRuntimeFilterCoveredBySplitDigest) { + ScopedConditionCacheForTest cache; + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->total_rows = ConditionCacheContext::GRANULE_SIZE; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, runtime_filter_wrapper_expr( + table_int32_greater_than_expr(0, 0, 0)))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .condition_cache_digest = 7, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + split_options.condition_cache_digest = 11; + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->condition_cache_ctx, nullptr); + EXPECT_FALSE(fake_state->condition_cache_ctx->is_hit); + + segment_v2::ConditionCacheHandle handle; + segment_v2::ConditionCache::ExternalCacheKey initial_digest_key( + "fake-table-reader-input", 0, -1, 7, 0, -1, + segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION); + EXPECT_FALSE(cache.get()->lookup(initial_digest_key, &handle)); + segment_v2::ConditionCache::ExternalCacheKey split_digest_key( + "fake-table-reader-input", 0, -1, 11, 0, -1, + segment_v2::ConditionCache::ExternalCacheKey::BASE_GRANULE_AWARE_VERSION); + EXPECT_TRUE(cache.get()->lookup(split_digest_key, &handle)); + ASSERT_TRUE(reader.close().ok()); +} + // Scenario: table-format delete files/deletion vectors are outside the data-file cache key. When // TableReader injects delete conjuncts into the file scan request, condition cache must be disabled // for that split. @@ -2411,6 +3326,7 @@ TEST(TableReaderTest, PushDownCountFromNewParquetReader) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); @@ -2452,6 +3368,7 @@ TEST(TableReaderTest, TableLevelCountUsesAssignedRowCount) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); auto split_options = build_split_options(file_path); @@ -2483,6 +3400,56 @@ TEST(TableReaderTest, TableLevelCountUsesAssignedRowCount) { std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, TableLevelCountRequiresExplicitCountStarArguments) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_table_count_arguments_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + const std::vector>> unsafe_count_arguments { + std::nullopt, std::vector {GlobalIndex(0)}}; + for (const auto& count_arguments : unsafe_count_arguments) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = count_arguments, + }) + .ok()); + auto split_options = build_split_options(file_path); + // Five metadata rows deliberately disagree with the three physical rows. nullopt models an + // old FE, while the non-empty vector models COUNT(id); neither may be treated as COUNT(*). + set_table_level_row_count(&split_options, 5); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + size_t total_rows = 0; + bool eos = false; + while (!eos) { + Block block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + total_rows += block.rows(); + } + EXPECT_EQ(3, total_rows); + ASSERT_TRUE(reader.close().ok()); + } + + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, PushDownMinMaxFromNewParquetReader) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_minmax_test"; std::filesystem::remove_all(test_dir); @@ -3299,6 +4266,7 @@ TEST(TableReaderTest, PushDownCountOnlyUsesSelectedRowGroupInFileRange) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options_for_row_group_mid(file_path, 2)).ok()); @@ -3338,6 +4306,7 @@ TEST(TableReaderTest, PushDownCountFallsBackWithTableConjunct) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); @@ -3379,6 +4348,7 @@ TEST(TableReaderTest, PushDownCountFallsBackWithFilter) { .runtime_state = &state, .scanner_profile = nullptr, .push_down_agg_type = TPushAggOp::type::COUNT, + .push_down_count_columns = std::vector {}, }) .ok()); ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); @@ -3605,6 +4575,59 @@ TEST(TableReaderTest, VExprPredicateSurvivesReopenSplit) { std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, RecomputesPredicateExecutionLayerForEverySplit) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_split_local_predicate_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto local_file = (test_dir / "local.parquet").string(); + const auto missing_file = (test_dir / "missing.parquet").string(); + write_single_int_parquet_file(local_file, "id", 3); + write_single_int_parquet_file(missing_file, "other", 9); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(local_file)).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + expect_int32_column_values(*block.get_by_position(0).column, {3}); + ASSERT_TRUE(reader.close().ok()); + + // The same predicate cannot be file-local when this split omits `id`. It must be rebuilt as a + // table-level predicate over the materialized NULL instead of inheriting the previous split's + // file-local ownership or escaping without exact evaluation. + ASSERT_TRUE(reader.prepare_split(build_split_options(missing_file)).ok()); + block = build_table_block(projected_columns); + eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 0); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) { const auto int_type = std::make_shared(); const std::vector projected_columns = { @@ -3625,14 +4648,13 @@ TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) { std::vector table_filters; table_filters.push_back({ - // This test only needs the referenced global indices to drive predicate-column - // placement. Keep the conjunct empty so the assertion focuses on scan-column - // de-duplication rather than expression rewrite/prepare behavior. - .conjunct = nullptr, + .conjunct = + VExprContext::create_shared(table_int32_sum_greater_than_expr(0, 0, 1, 1, 1)), .global_indices = {GlobalIndex(0), GlobalIndex(1)}, }); table_filters.push_back({ - .conjunct = nullptr, + .conjunct = + VExprContext::create_shared(table_int32_sum_greater_than_expr(0, 0, 2, 2, 1)), .global_indices = {GlobalIndex(0), GlobalIndex(2)}, }); @@ -3680,6 +4702,7 @@ TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) EXPECT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); EXPECT_EQ(projection_ids(file_request.non_predicate_columns), std::vector({1})); + EXPECT_TRUE(file_request.predicate_only_columns.empty()); ASSERT_EQ(file_request.local_positions.size(), 2); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(0)).value(), 1); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(1)).value(), 0); @@ -3975,7 +4998,7 @@ TEST(TableReaderTest, ProjectedColumnsFillMissingParquetColumnWithDefault) { std::filesystem::remove_all(test_dir); } -TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { +TEST(TableReaderTest, ProjectedStructFillsMissingChildWithBinaryInitialDefault) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_struct_missing_child_test"; std::filesystem::remove_all(test_dir); @@ -3985,10 +5008,12 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { write_struct_parquet_file(file_path, 7); const auto int_type = std::make_shared(); - const auto string_type = std::make_shared(); + const auto varbinary_type = std::make_shared(16); auto id_child = make_table_column(0, "id", int_type); - auto missing_child = make_table_column(99, "missing_child", string_type); - auto struct_type = std::make_shared(DataTypes {int_type, string_type}, + auto missing_child = make_table_column(99, "missing_child", varbinary_type); + missing_child.initial_default_value = "Ej5FZ+ibEtOkVkJmFBdAAA=="; + missing_child.initial_default_value_is_base64 = true; + auto struct_type = std::make_shared(DataTypes {int_type, varbinary_type}, Strings {"id", "missing_child"}); auto struct_column = make_table_column(100, "s", struct_type); struct_column.children = {id_child, missing_child}; @@ -4022,7 +5047,10 @@ TEST(TableReaderTest, ProjectedStructFillsMissingChildWithDefault) { expect_not_null_nullable_nested_column(struct_result.get_column(0))); ASSERT_EQ(struct_result.size(), 1); EXPECT_EQ(ids.get_element(0), 7); - expect_nullable_column_all_null(struct_result.get_column(1)); + const auto& defaults = assert_cast( + expect_not_null_nullable_nested_column(struct_result.get_column(1))); + EXPECT_EQ(defaults.get_data_at(0).to_string(), + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16)); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index c2aea1cd825253..975f1e8eddd88b 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -54,6 +54,16 @@ io::FileCacheStatistics make_file_cache_stats(int64_t multiplier) { stats.inverted_index_io_timer = multiplier * 28; stats.remote_only_on_miss_triggered = multiplier * 29; stats.remote_only_on_miss_threshold_bytes = multiplier * 30; + stats.num_cross_cg_peer_io_total = multiplier * 31; + stats.bytes_read_from_cross_cg_peer = multiplier * 32; + stats.cross_cg_peer_io_timer = multiplier * 33; + stats.num_same_cg_peer_io_total = multiplier * 34; + stats.bytes_read_from_same_cg_peer = multiplier * 35; + stats.same_cg_peer_io_timer = multiplier * 36; + stats.num_peer_race_peer_win = multiplier * 37; + stats.num_peer_race_s3_win = multiplier * 38; + stats.num_peer_lazy_fetch = multiplier * 39; + stats.peer_lazy_fetch_timer = multiplier * 40; return stats; } @@ -94,6 +104,16 @@ void expect_file_cache_stats_eq(const io::FileCacheStatistics& actual, EXPECT_EQ(actual.remote_only_on_miss_triggered, expected.remote_only_on_miss_triggered); EXPECT_EQ(actual.remote_only_on_miss_threshold_bytes, expected.remote_only_on_miss_threshold_bytes); + EXPECT_EQ(actual.num_cross_cg_peer_io_total, expected.num_cross_cg_peer_io_total); + EXPECT_EQ(actual.bytes_read_from_cross_cg_peer, expected.bytes_read_from_cross_cg_peer); + EXPECT_EQ(actual.cross_cg_peer_io_timer, expected.cross_cg_peer_io_timer); + EXPECT_EQ(actual.num_same_cg_peer_io_total, expected.num_same_cg_peer_io_total); + EXPECT_EQ(actual.bytes_read_from_same_cg_peer, expected.bytes_read_from_same_cg_peer); + EXPECT_EQ(actual.same_cg_peer_io_timer, expected.same_cg_peer_io_timer); + EXPECT_EQ(actual.num_peer_race_peer_win, expected.num_peer_race_peer_win); + EXPECT_EQ(actual.num_peer_race_s3_win, expected.num_peer_race_s3_win); + EXPECT_EQ(actual.num_peer_lazy_fetch, expected.num_peer_lazy_fetch); + EXPECT_EQ(actual.peer_lazy_fetch_timer, expected.peer_lazy_fetch_timer); } } // namespace @@ -139,6 +159,10 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot EXPECT_EQ(profile->get_counter("CacheGetOrSetTimer")->value(), after_second_report.cache_get_or_set_timer); EXPECT_EQ(profile->get_counter("LockWaitTimer")->value(), after_second_report.lock_wait_timer); + EXPECT_EQ(profile->get_counter("CrossCGPeerIOTime")->value(), + after_second_report.cross_cg_peer_io_timer); + EXPECT_EQ(profile->get_counter("PeerLazyFetchTime")->value(), + after_second_report.peer_lazy_fetch_timer); } } // namespace doris diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp index 089b90e12ba90a..9b35ae10a325d1 100644 --- a/be/test/io/cache/block_file_cache_test.cpp +++ b/be/test/io/cache/block_file_cache_test.cpp @@ -4108,6 +4108,73 @@ TEST_F(BlockFileCacheTest, test_factory_1) { FileCacheFactory::instance()->_capacity = 0; } +TEST_F(BlockFileCacheTest, create_file_caches_preserves_config_order) { + reset_file_cache_factory_for_test(); + std::string cache_path2 = caches_dir / "cache2" / ""; + std::string cache_path3 = caches_dir / "cache3" / ""; + if (fs::exists(cache_base_path)) { + fs::remove_all(cache_base_path); + } + if (fs::exists(cache_path2)) { + fs::remove_all(cache_path2); + } + if (fs::exists(cache_path3)) { + fs::remove_all(cache_path3); + } + Defer cleanup {[&] { + reset_file_cache_factory_for_test(); + if (fs::exists(cache_base_path)) { + fs::remove_all(cache_base_path); + } + if (fs::exists(cache_path2)) { + fs::remove_all(cache_path2); + } + if (fs::exists(cache_path3)) { + fs::remove_all(cache_path3); + } + }}; + + std::vector cache_paths; + cache_paths.emplace_back(cache_base_path, 90, 30, DEFAULT_NORMAL_PERCENT, + DEFAULT_DISPOSABLE_PERCENT, DEFAULT_INDEX_PERCENT, DEFAULT_TTL_PERCENT, + "disk"); + cache_paths.emplace_back(cache_path2, 120, 30, DEFAULT_NORMAL_PERCENT, + DEFAULT_DISPOSABLE_PERCENT, DEFAULT_INDEX_PERCENT, DEFAULT_TTL_PERCENT, + "disk"); + cache_paths.emplace_back(cache_base_path, 90, 30, DEFAULT_NORMAL_PERCENT, + DEFAULT_DISPOSABLE_PERCENT, DEFAULT_INDEX_PERCENT, DEFAULT_TTL_PERCENT, + "disk"); + cache_paths.emplace_back(cache_path3, 150, 30, DEFAULT_NORMAL_PERCENT, + DEFAULT_DISPOSABLE_PERCENT, DEFAULT_INDEX_PERCENT, DEFAULT_TTL_PERCENT, + "disk"); + + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_caches(cache_paths, [](const std::string&, + const Status&) { return false; }) + .ok()); + + const auto& caches = FileCacheFactory::instance()->get_caches(); + ASSERT_EQ(caches.size(), 3); + EXPECT_EQ(caches[0]->get_base_path(), cache_base_path); + EXPECT_EQ(caches[1]->get_base_path(), cache_path2); + EXPECT_EQ(caches[2]->get_base_path(), cache_path3); + EXPECT_EQ(FileCacheFactory::instance()->get_by_path(cache_base_path)->get_base_path(), + cache_base_path); + EXPECT_EQ(FileCacheFactory::instance()->get_by_path(cache_path2)->get_base_path(), cache_path2); + EXPECT_EQ(FileCacheFactory::instance()->get_by_path(cache_path3)->get_base_path(), cache_path3); + EXPECT_EQ(FileCacheFactory::instance()->get_capacity(), 360); + + for (const auto& cache : caches) { + wait_until_cache_ready(*cache); + } + + for (int i = 0; i < 64; ++i) { + auto key = io::BlockFileCache::hash("factory_order_key_" + std::to_string(i)); + const auto expected_path = caches[KeyHash()(key) % caches.size()]->get_base_path(); + EXPECT_EQ(FileCacheFactory::instance()->get_by_path(key)->get_base_path(), expected_path); + } +} + TEST_F(BlockFileCacheTest, test_factory_2) { if (fs::exists(cache_base_path)) { fs::remove_all(cache_base_path); @@ -5935,7 +6002,7 @@ TEST_F(BlockFileCacheTest, test_align_size) { std::random_device rd; // a seed source for the random number engine std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd() std::uniform_int_distribution<> distrib(0, 10_mb + 10086); - std::ranges::for_each(std::ranges::iota_view {0, 1000}, [&](int) { + for (int loop_i = 0; loop_i < 1000; ++loop_i) { size_t read_size = distrib(gen) % 1_mb; size_t read_offset = distrib(gen); auto [offset, size] = @@ -5943,7 +6010,7 @@ TEST_F(BlockFileCacheTest, test_align_size) { EXPECT_EQ(offset % 1_mb, 0); EXPECT_GE(size, 1_mb); EXPECT_LE(size, 2_mb); - }); + } } TEST_F(BlockFileCacheTest, remove_if_cached_when_isnt_releasable) { @@ -6045,7 +6112,7 @@ TEST_F(BlockFileCacheTest, cached_remote_file_reader_opt_lock) { std::random_device rd; // a seed source for the random number engine std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd() std::uniform_int_distribution<> distrib(1_mb, 7_mb); - std::ranges::for_each(std::ranges::iota_view {0, 1000}, [&](int) { + for (int loop_i = 0; loop_i < 1000; ++loop_i) { size_t read_offset = distrib(gen); size_t read_size = distrib(gen) % 1_mb; if (read_offset + read_size > 7_mb || read_size == 0) { @@ -6069,7 +6136,7 @@ TEST_F(BlockFileCacheTest, cached_remote_file_reader_opt_lock) { } else { EXPECT_EQ(std::string(read_size, '0' + num), buffer); } - }); + } } { FileReaderSPtr local_reader; diff --git a/be/test/io/cache/block_file_cache_test_common.h b/be/test/io/cache/block_file_cache_test_common.h index 17de4bb814b51e..a8311e52e181a6 100644 --- a/be/test/io/cache/block_file_cache_test_common.h +++ b/be/test/io/cache/block_file_cache_test_common.h @@ -45,7 +45,6 @@ #include #include #include -#include #include #include #include diff --git a/be/test/io/cache/block_file_cache_test_meta_store.cpp b/be/test/io/cache/block_file_cache_test_meta_store.cpp index a658e02d17d2b6..324feed267d34c 100644 --- a/be/test/io/cache/block_file_cache_test_meta_store.cpp +++ b/be/test/io/cache/block_file_cache_test_meta_store.cpp @@ -277,6 +277,7 @@ TEST_F(BlockFileCacheTest, version3_add_remove_restart) { ASSERT_EQ(cache._lru_recorder->_shadow_disposable_queue.get_elements_num_unsafe(), 2); EXPECT_EQ(cache.replay_lru_logs_once(), 0); EXPECT_EQ(cache._lru_recorder_log_replay_idle_metrics->get_value(), 1); + cache.dump_lru_queues(true); // check the meta store to see the content { @@ -312,8 +313,6 @@ TEST_F(BlockFileCacheTest, version3_add_remove_restart) { verify_meta_key(*meta_store, 50, "key4", 300000, FileCacheType::DISPOSABLE, 0, 100000); verify_meta_key(*meta_store, 50, "key4", 400000, FileCacheType::DISPOSABLE, 0, 100000); } - std::this_thread::sleep_for( - std::chrono::milliseconds(2 * config::file_cache_background_lru_dump_interval_ms)); } { // cache2 diff --git a/be/test/io/cache/cache_lru_dumper_test.cpp b/be/test/io/cache/cache_lru_dumper_test.cpp index 328b2b7e7de8ee..6256f52d74a8be 100644 --- a/be/test/io/cache/cache_lru_dumper_test.cpp +++ b/be/test/io/cache/cache_lru_dumper_test.cpp @@ -160,6 +160,48 @@ TEST_F(CacheLRUDumperTest, test_dump_and_restore_queue) { } } +TEST_F(CacheLRUDumperTest, test_parse_multiple_groups_without_mutating_repeated_fields) { + const auto old_tail_record_num = config::file_cache_background_lru_dump_tail_record_num; + Defer defer {[old_tail_record_num] { + config::file_cache_background_lru_dump_tail_record_num = old_tail_record_num; + }}; + config::file_cache_background_lru_dump_tail_record_num = 10001; + + LRUQueue src_queue; + std::string queue_name = "normal"; + std::lock_guard lock(_mutex); + UInt128Wrapper hash(987654321ULL); + + for (size_t i = 0; i < 10001; ++i) { + src_queue.add(hash, i * 4096, 4096 + i, lock); + } + + dumper->do_dump_queue(src_queue, queue_name); + + std::string filename = fmt::format("{}lru_dump_{}.tail", test_dir, queue_name); + std::ifstream in(filename, std::ios::binary); + ASSERT_TRUE(in); + + size_t entry_num = 0; + ASSERT_TRUE(dumper->parse_dump_footer(in, filename, entry_num).ok()); + ASSERT_EQ(entry_num, 10001); + ASSERT_EQ(dumper->_parse_meta.group_offset_size_size(), 2); + + UInt128Wrapper parsed_hash; + size_t offset = 0; + size_t size = 0; + for (size_t i = 0; i < entry_num; ++i) { + ASSERT_TRUE(dumper->parse_one_lru_entry(in, filename, parsed_hash, offset, size).ok()); + EXPECT_EQ(parsed_hash, hash); + EXPECT_EQ(offset, i * 4096); + EXPECT_EQ(size, 4096 + i); + } + + EXPECT_EQ(dumper->_parse_meta.group_offset_size_size(), 2); + EXPECT_EQ(dumper->_parse_group_index, 2); + EXPECT_EQ(dumper->_parse_entry_index, 1); +} + TEST_F(CacheLRUDumperTest, test_lru_log_record_disabled_keeps_existing_backlog) { const auto old_tail_record_num = config::file_cache_background_lru_dump_tail_record_num; const auto old_queue_limit = config::file_cache_background_lru_log_queue_max_size; diff --git a/be/test/io/cache/lru_queue_test.cpp b/be/test/io/cache/lru_queue_test.cpp index 2a9cdc3a6bc672..36aed888184d59 100644 --- a/be/test/io/cache/lru_queue_test.cpp +++ b/be/test/io/cache/lru_queue_test.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include diff --git a/be/test/io/fs/stream_sink_file_writer_test.cpp b/be/test/io/fs/stream_sink_file_writer_test.cpp index 11de185d8e2e2d..35d9fb1ef68b1c 100644 --- a/be/test/io/fs/stream_sink_file_writer_test.cpp +++ b/be/test/io/fs/stream_sink_file_writer_test.cpp @@ -20,10 +20,15 @@ #include #include +#include +#include + +#include "common/config.h" #include "exec/sink/load_stream_stub.h" #include "gtest/gtest_pred_impl.h" #include "storage/olap_common.h" #include "util/debug/leakcheck_disabler.h" +#include "util/debug_points.h" #include "util/faststring.h" namespace doris { @@ -47,13 +52,16 @@ const std::string DATA0 = "segment data"; const std::string DATA1 = "hello world"; static std::atomic g_num_request; +static std::unordered_map g_num_requests_by_dst; class StreamSinkFileWriterTest : public testing::Test { class MockStreamStub : public LoadStreamStub { public: - MockStreamStub(PUniqueId load_id, int64_t src_id) + MockStreamStub(PUniqueId load_id, int64_t src_id, int64_t dst_id) : LoadStreamStub(load_id, src_id, std::make_shared(), - std::make_shared()) {}; + std::make_shared()) { + _dst_id = dst_id; + } virtual ~MockStreamStub() = default; @@ -76,6 +84,7 @@ class StreamSinkFileWriterTest : public testing::Test { EXPECT_EQ(0, offset); } g_num_request++; + g_num_requests_by_dst[_dst_id]++; return Status::OK(); } }; @@ -88,15 +97,30 @@ class StreamSinkFileWriterTest : public testing::Test { virtual void SetUp() { _load_id.set_hi(LOAD_ID_HI); _load_id.set_lo(LOAD_ID_LO); + std::array dst_ids {103, 101, 102}; for (int src_id = 0; src_id < NUM_STREAM; src_id++) { - _streams.emplace_back(new MockStreamStub(_load_id, src_id)); + _streams.emplace_back(new MockStreamStub(_load_id, src_id, dst_ids[src_id])); } + _enable_debug_points = config::enable_debug_points; + g_num_requests_by_dst.clear(); } - virtual void TearDown() {} + virtual void TearDown() { + DebugPoints::instance()->clear(); + config::enable_debug_points = _enable_debug_points; + } + + void write_with_streams(const std::vector>& streams) { + io::StreamSinkFileWriter writer(streams); + writer.init(_load_id, PARTITION_ID, INDEX_ID, TABLET_ID, SEGMENT_ID); + std::vector slices {DATA0, DATA1}; + CHECK_STATUS_OK(writer.appendv(&(*slices.begin()), slices.size())); + CHECK_STATUS_OK(writer.close()); + } PUniqueId _load_id; std::vector> _streams; + bool _enable_debug_points; }; TEST_F(StreamSinkFileWriterTest, Test) { @@ -111,4 +135,28 @@ TEST_F(StreamSinkFileWriterTest, Test) { EXPECT_EQ(NUM_STREAM * 2, g_num_request); } +TEST_F(StreamSinkFileWriterTest, DeterministicOneReplicaFaultInjection) { + config::enable_debug_points = true; + DebugPoints::instance()->add("StreamSinkFileWriter.appendv.write_segment_failed_one_replica"); + + write_with_streams(_streams); + write_with_streams({_streams[2], _streams[0], _streams[1]}); + + EXPECT_EQ(0, g_num_requests_by_dst[101]); + EXPECT_EQ(4, g_num_requests_by_dst[102]); + EXPECT_EQ(4, g_num_requests_by_dst[103]); +} + +TEST_F(StreamSinkFileWriterTest, DeterministicTwoReplicaFaultInjection) { + config::enable_debug_points = true; + DebugPoints::instance()->add("StreamSinkFileWriter.appendv.write_segment_failed_two_replica"); + + write_with_streams(_streams); + write_with_streams({_streams[2], _streams[0], _streams[1]}); + + EXPECT_EQ(0, g_num_requests_by_dst[101]); + EXPECT_EQ(0, g_num_requests_by_dst[102]); + EXPECT_EQ(4, g_num_requests_by_dst[103]); +} + } // namespace doris diff --git a/be/test/olap/rowset/group_rowset_writer_test.cpp b/be/test/olap/rowset/group_rowset_writer_test.cpp index 4ebd2db4ca3234..edf616f323b9e5 100644 --- a/be/test/olap/rowset/group_rowset_writer_test.cpp +++ b/be/test/olap/rowset/group_rowset_writer_test.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -36,6 +37,9 @@ #include "runtime/runtime_profile.h" #include "storage/binlog.h" #include "storage/olap_define.h" +#include "storage/partial_update_info.h" +#include "storage/rowset/rowset_reader.h" +#include "storage/rowset/rowset_reader_context.h" #include "storage/storage_engine.h" #include "storage/tablet/tablet.h" #include "storage/tablet/tablet_manager.h" @@ -72,10 +76,25 @@ class GroupRowsetWriterTest : public testing::Test { ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); _request = testutil::create_tablet_request( - 10010, 270068390, 10001, 1, TKeysType::UNIQUE_KEYS, - {{"k1", TPrimitiveType::INT, true}, {"v1", TPrimitiveType::INT, false}}); + 10010, 270068390, 10001, 2, TKeysType::UNIQUE_KEYS, + {{"k1", TPrimitiveType::INT, true}, + {"__DORIS_TEST_HIDDEN_KEY__", TPrimitiveType::BIGINT, true}, + {"__DORIS_TEST_HIDDEN_VALUE__", TPrimitiveType::INT, false}, + {"v1", TPrimitiveType::INT, false}, + {"v2", TPrimitiveType::BIGINT, false}, + {std::string(DELETE_SIGN), TPrimitiveType::TINYINT, false}}); + _request.tablet_schema.columns[1].__set_visible(false); + _request.tablet_schema.columns[2].__set_visible(false); + _request.tablet_schema.columns[5].__set_visible(false); + _request.tablet_schema.columns[2].__set_is_allow_null(true); + _request.tablet_schema.columns[4].__set_is_allow_null(true); _request.__set_enable_unique_key_merge_on_write(true); testutil::enable_row_binlog(&_request); + _request.row_binlog_schema.columns.erase(_request.row_binlog_schema.columns.begin() + 5); + _request.row_binlog_schema.columns.erase(_request.row_binlog_schema.columns.begin() + 2); + _request.row_binlog_schema.__set_binlog_tso_idx(4); + _request.row_binlog_schema.__set_binlog_lsn_idx(5); + _request.row_binlog_schema.__set_binlog_op_idx(6); auto profile = std::make_unique("GroupRowsetWriterTest"); ASSERT_TRUE(engine_ptr->create_tablet(_request, profile.get()).ok()); _tablet = engine_ptr->tablet_manager()->get_tablet(_request.tablet_id); @@ -101,13 +120,31 @@ class GroupRowsetWriterTest : public testing::Test { auto& columns = columns_guard.mutable_columns(); for (int i = 0; i < num_rows; ++i) { columns[0]->insert(Field::create_field(start_key + i)); - columns[1]->insert( + columns[1]->insert(Field::create_field( + static_cast(1000 + start_key + i))); + columns[2]->insert(Field::create_field( + static_cast(10000 + start_key + i))); + columns[3]->insert( Field::create_field((start_key + i) * 10)); + columns[4]->insert(Field::create_field( + static_cast((start_key + i) * 100))); + columns[5]->insert(Field::create_field(0)); } } return block; } + Block create_partial_update_block() const { + Block block = _tablet->tablet_schema()->create_block_by_cids({0, 1, 2, 3}); + auto columns_guard = block.mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + columns[0]->insert(Field::create_field(1)); + columns[1]->insert(Field::create_field(1001)); + columns[2]->insert(Field::create_field(200)); + columns[3]->insert(Field::create_field(20)); + return block; + } + Status create_group_rowset_writer(std::unique_ptr* group_writer, RowsetId* data_rowset_id, size_t num_rows) { RowsetWriterContext data_context; @@ -225,4 +262,87 @@ TEST_F(GroupRowsetWriterTest, success) { EXPECT_TRUE(file_exists(second_segment_path)); } +TEST_F(GroupRowsetWriterTest, partialUpdateSkipsHiddenNonKeyColumns) { + ASSERT_EQ(1, _tablet->row_binlog_tablet_schema()->field_index("__DORIS_TEST_HIDDEN_KEY__")); + ASSERT_EQ(-1, _tablet->row_binlog_tablet_schema()->field_index("__DORIS_TEST_HIDDEN_VALUE__")); + ASSERT_EQ(-1, _tablet->row_binlog_tablet_schema()->field_index(DELETE_SIGN)); + + auto partial_update_info = std::make_shared(); + ASSERT_TRUE( + partial_update_info + ->init(_tablet->tablet_id(), 1, *_tablet->tablet_schema(), + UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS, + PartialUpdateNewRowPolicyPB::APPEND, + {"k1", "__DORIS_TEST_HIDDEN_KEY__", "__DORIS_TEST_HIDDEN_VALUE__", "v1"}, + false, 0, 0, "", "") + .ok()); + EXPECT_EQ((std::vector {0, 1, 2, 3}), partial_update_info->update_cids); + EXPECT_EQ((std::vector {4, 5}), partial_update_info->missing_cids); + + RowsetWriterContext row_binlog_context; + row_binlog_context.tablet = _tablet; + row_binlog_context.tablet_schema = _tablet->row_binlog_tablet_schema(); + row_binlog_context.rowset_state = PREPARED; + row_binlog_context.segments_overlap = NONOVERLAPPING; + row_binlog_context.max_rows_per_segment = 1024; + row_binlog_context.write_type = DataWriteType::TYPE_DIRECT; + row_binlog_context.partial_update_info = partial_update_info; + row_binlog_context.mow_context = + std::make_shared(1, 1, std::make_shared(), + std::vector {}, nullptr); + row_binlog_context.write_binlog_opt().enable = true; + auto& binlog_options = row_binlog_context.write_binlog_opt().write_binlog_config(); + binlog_options.source.tablet_schema = _tablet->tablet_schema(); + binlog_options.source.partial_update_info = partial_update_info; + binlog_options.source.mow_context = row_binlog_context.mow_context; + binlog_options.source.source_write_type = DataWriteType::TYPE_DIRECT; + + auto lsn_buffer = AutoIncIDBuffer::create_shared(1, 1, kBinlogLsnAutoIncId); + lsn_buffer->append_range_for_test(1000, 1); + auto lsn_ids = std::make_shared>(); + ASSERT_TRUE(allocate_binlog_lsn(lsn_buffer, 1, *lsn_ids).ok()); + binlog_options.insert_seg_lsn(0, lsn_ids); + + auto row_binlog_writer_res = _tablet->create_rowset_writer(row_binlog_context, false); + ASSERT_TRUE(row_binlog_writer_res.has_value()); + auto row_binlog_writer = std::move(row_binlog_writer_res.value()); + + Block block = create_partial_update_block(); + ASSERT_TRUE(row_binlog_writer->flush_single_block(&block).ok()); + + RowsetSharedPtr row_binlog_rowset; + ASSERT_TRUE(row_binlog_writer->build(row_binlog_rowset).ok()); + ASSERT_EQ(1, row_binlog_rowset->num_segments()); + + const auto& row_binlog_schema = _tablet->row_binlog_tablet_schema(); + ASSERT_EQ(7, row_binlog_schema->num_columns()); + std::vector return_columns {0, 1, 2, 3, 4, 5, 6}; + RowsetReaderContext reader_context; + reader_context.tablet_schema = row_binlog_schema; + reader_context.need_ordered_result = false; + reader_context.return_columns = &return_columns; + + RowsetReaderSharedPtr rowset_reader; + ASSERT_TRUE(row_binlog_rowset->create_reader(&rowset_reader).ok()); + ASSERT_TRUE(rowset_reader->init(&reader_context).ok()); + + Block output_block = row_binlog_schema->create_block(); + auto status = rowset_reader->next_batch(&output_block); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(1, output_block.rows()); + ASSERT_EQ(return_columns.size(), output_block.columns()); + + EXPECT_EQ(1, (*output_block.get_by_position(0).column)[0].get()); + EXPECT_EQ(1001, (*output_block.get_by_position(1).column)[0].get()); + EXPECT_EQ(20, (*output_block.get_by_position(2).column)[0].get()); + EXPECT_TRUE(output_block.get_by_position(3).column->is_null_at(0)); + EXPECT_TRUE(output_block.get_by_position(4).column->is_null_at(0)); + EXPECT_EQ(1000, (*output_block.get_by_position(5).column)[0].get()); + EXPECT_EQ(ROW_BINLOG_APPEND, (*output_block.get_by_position(6).column)[0].get()); + + Block eof_block = row_binlog_schema->create_block(); + status = rowset_reader->next_batch(&eof_block); + EXPECT_TRUE(status.is()) << status; +} + } // namespace doris diff --git a/be/test/runtime/runtime_profile_counter_tree_node_test.cpp b/be/test/runtime/runtime_profile_counter_tree_node_test.cpp index 6e60eb4840821c..e3e40aa8c7cc3e 100644 --- a/be/test/runtime/runtime_profile_counter_tree_node_test.cpp +++ b/be/test/runtime/runtime_profile_counter_tree_node_test.cpp @@ -167,10 +167,10 @@ TEST_F(RuntimeProfileCounterTreeNodeTest, HighWaterMarkCounterToThrift) { childCounterMap["root"].insert("child"); rootCounter->add(10); - rootCounter->set(5); + rootCounter->set(int64_t(5)); childCounter->add(100); - childCounter->set(50); + childCounter->set(int64_t(50)); RuntimeProfileCounterTreeNode rootNode = RuntimeProfileCounterTreeNode::from_map( counterMap, childCounterMap, RuntimeProfile::ROOT_COUNTER); diff --git a/be/test/runtime/runtime_profile_test.cpp b/be/test/runtime/runtime_profile_test.cpp index 7e36867805a6c5..2734084e9cb202 100644 --- a/be/test/runtime/runtime_profile_test.cpp +++ b/be/test/runtime/runtime_profile_test.cpp @@ -76,7 +76,7 @@ TEST(RuntimeProfileTest, Basic) { counter_a->update(10); counter_a->update(-5); EXPECT_EQ(counter_a->value(), 5); - counter_a->set(1L); + counter_a->set(int64_t(1)); EXPECT_EQ(counter_a->value(), 1); counter_b = profile_a2.add_counter("B", TUnit::BYTES); @@ -149,7 +149,7 @@ TEST(RuntimeProfileTest, ProtoBasic) { counter_a->update(10); counter_a->update(-5); EXPECT_EQ(counter_a->value(), 5); - counter_a->set(1L); + counter_a->set(int64_t(1)); EXPECT_EQ(counter_a->value(), 1); counter_b = profile_a2.add_counter("B", TUnit::BYTES); @@ -441,7 +441,7 @@ TEST(RuntimeProfileTest, DerivedCounters) { RuntimeProfile::Counter* bytes_counter = profile.add_counter("bytes", TUnit::BYTES); RuntimeProfile::Counter* ticks_counter = profile.add_counter("ticks", TUnit::TIME_NS); // set to 1 sec - ticks_counter->set(1000L * 1000L * 1000L); + ticks_counter->set(int64_t(1000L * 1000L * 1000L)); RuntimeProfile::DerivedCounter* throughput_counter = profile.add_derived_counter( "throughput", TUnit::BYTES, @@ -450,9 +450,9 @@ TEST(RuntimeProfileTest, DerivedCounters) { }, RuntimeProfile::ROOT_COUNTER); - bytes_counter->set(10L); + bytes_counter->set(int64_t(10)); EXPECT_EQ(throughput_counter->value(), 10); - bytes_counter->set(20L); + bytes_counter->set(int64_t(20)); EXPECT_EQ(throughput_counter->value(), 20); ticks_counter->set(ticks_counter->value() / 2); EXPECT_EQ(throughput_counter->value(), 40); diff --git a/be/test/storage/compaction/compaction_permit_limiter_test.cpp b/be/test/storage/compaction/compaction_permit_limiter_test.cpp index 8dba493c9ccc92..f6595608fe6479 100644 --- a/be/test/storage/compaction/compaction_permit_limiter_test.cpp +++ b/be/test/storage/compaction/compaction_permit_limiter_test.cpp @@ -23,6 +23,8 @@ #include #include +#include + #include "common/config.h" namespace doris { diff --git a/be/test/storage/index/ann/ann_index_edge_case_test.cpp b/be/test/storage/index/ann/ann_index_edge_case_test.cpp index a42085f02c3599..e6f43b7ba4fed7 100644 --- a/be/test/storage/index/ann/ann_index_edge_case_test.cpp +++ b/be/test/storage/index/ann/ann_index_edge_case_test.cpp @@ -43,8 +43,8 @@ TEST_F(VectorSearchTest, TestAnnIndexStatsInitialization) { EXPECT_EQ(stats.range_fallback_small_candidate_rows, 0); // Test setting values - stats.search_costs_ns.set(1000L); - stats.load_index_costs_ns.set(2000L); + stats.search_costs_ns.set(int64_t(1000)); + stats.load_index_costs_ns.set(int64_t(2000)); stats.range_fallback_by_small_candidate_cnt = 1; stats.range_fallback_small_candidate_rows = 3; @@ -56,8 +56,8 @@ TEST_F(VectorSearchTest, TestAnnIndexStatsInitialization) { TEST_F(VectorSearchTest, TestAnnIndexStatsCopyConstructor) { doris::segment_v2::AnnIndexStats original; - original.search_costs_ns.set(1500L); - original.load_index_costs_ns.set(2500L); + original.search_costs_ns.set(int64_t(1500)); + original.load_index_costs_ns.set(int64_t(2500)); original.range_fallback_by_small_candidate_cnt = 1; original.range_fallback_small_candidate_rows = 3; diff --git a/be/test/storage/iterator/block_reader_change_next_block_test.cpp b/be/test/storage/iterator/block_reader_change_next_block_test.cpp new file mode 100644 index 00000000000000..bfa774c325cec9 --- /dev/null +++ b/be/test/storage/iterator/block_reader_change_next_block_test.cpp @@ -0,0 +1,483 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// End-to-end branch coverage for BlockReader::_min_delta_next_block and +// BlockReader::_detail_change_next_block. Instead of standing up a real tablet / +// rowset stack, we inject a fake VCollectIterator::LevelIterator that walks a +// pre-built merged binlog block one row at a time, so we can exercise every +// min-delta fold result (SKIP / INSERT / DELETE / UPDATE_BEFORE_AFTER) and every +// detail op (INSERT / DELETE / UPDATE, including the pending-row batch split). + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wkeyword-macro" +#endif +#define private public +#define protected public +#include "storage/iterator/block_reader.h" +#include "storage/iterator/vcollect_iterator.h" +#undef private +#undef protected +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +#include + +#include +#include +#include + +#include "common/config.h" +#include "common/status.h" +#include "core/assert_cast.h" +#include "core/block/block.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "storage/binlog.h" +#include "storage/iterator/binlog_block_reader_utils.h" +#include "storage/tablet/tablet_schema.h" +#include "storage/utils.h" + +namespace doris { + +using namespace ErrorCode; + +namespace { + +// Merged binlog block schema used across the tests. Column layout mirrors what a +// row-binlog scan produces after merge: +// 0: key (Int64, the primary key used to group same-key rows) +// 1: val (Int64, the "after" value of a data column) +// 2: __BEFORE__val__ (Int64, the "before" value mirror of `val`) +// 3: __DORIS_BINLOG_TSO__ (Int64) +// 4: __DORIS_BINLOG_LSN__ (Int64) +// 5: __DORIS_BINLOG_OP__ (Int64, one of ROW_BINLOG_APPEND/UPDATE/DELETE) +constexpr int KEY_IDX = 0; +constexpr int VAL_IDX = 1; +constexpr int BEFORE_VAL_IDX = 2; +constexpr int TSO_IDX = 3; +constexpr int LSN_IDX = 4; +constexpr int OP_IDX = 5; + +struct Row { + int64_t key; + int64_t val; + int64_t before_val; + int64_t tso; + int64_t lsn; + int64_t op; +}; + +std::shared_ptr make_source_block(const std::vector& rows) { + auto block = std::make_shared(); + auto type = std::make_shared(); + + auto key_col = ColumnInt64::create(); + auto val_col = ColumnInt64::create(); + auto before_col = ColumnInt64::create(); + auto tso_col = ColumnInt64::create(); + auto lsn_col = ColumnInt64::create(); + auto op_col = ColumnInt64::create(); + for (const auto& r : rows) { + key_col->insert_value(r.key); + val_col->insert_value(r.val); + before_col->insert_value(r.before_val); + tso_col->insert_value(r.tso); + lsn_col->insert_value(r.lsn); + op_col->insert_value(r.op); + } + block->insert({std::move(key_col), type, "key"}); + block->insert({std::move(val_col), type, "val"}); + block->insert({std::move(before_col), type, binlog::build_before_column_name("val")}); + block->insert({std::move(tso_col), type, BINLOG_TSO_COL}); + block->insert({std::move(lsn_col), type, BINLOG_LSN_COL}); + block->insert({std::move(op_col), type, BINLOG_OP_COL}); + return block; +} + +// Fake merge iterator: hands out rows from `source` one at a time. `is_same` is +// derived from primary-key equality with the previous emitted row, matching the +// real merge iterator's contract that consecutive same-key rows are flagged. +class FakeLevelIterator : public VCollectIterator::LevelIterator { +public: + FakeLevelIterator(TabletReader* reader, std::shared_ptr source) + : LevelIterator(reader), _source(std::move(source)) { + _ref.block = _source; + _ref.row_pos = 0; + _ref.is_same = false; + } + + Status init(bool /*get_data_by_ref*/) override { return Status::OK(); } + int64_t version() const override { return 0; } + + Status next(IteratorRowRef* ref) override { + int prev = _ref.row_pos; + int next_pos = prev + 1; + if (next_pos >= _source->rows()) { + _ref.row_pos = -1; + *ref = _ref; + return Status::Error(""); + } + int64_t prev_key = read_i64(KEY_IDX, prev); + int64_t cur_key = read_i64(KEY_IDX, next_pos); + _ref.row_pos = next_pos; + _ref.is_same = (prev_key == cur_key); + *ref = _ref; + return Status::OK(); + } + + Status next(Block* /*block*/) override { return Status::Error(""); } + + RowLocation current_row_location() override { return RowLocation(); } + Status current_block_row_locations(std::vector* /*loc*/) override { + return Status::OK(); + } + Status ensure_first_row_ref() override { return Status::OK(); } + void update_profile(RuntimeProfile* /*profile*/) override {} + +private: + int64_t read_i64(int col, int row) const { + return assert_cast(*_source->get_by_position(col).column) + .get_element(row); + } + + std::shared_ptr _source; +}; + +// Wire a BlockReader as if init() had already completed for a row-binlog change +// scan over the fixed 6-column schema, then plug in the fake merge iterator. +void configure_reader(BlockReader& reader, std::shared_ptr source, size_t batch_size) { + config::enable_adaptive_batch_size = false; + reader._reader_context.batch_size = batch_size; + + // The fake LevelIterator base ctor dereferences reader->tablet_schema(), so a + // schema must exist even though its contents are unused by these code paths. + reader._tablet_schema = std::make_shared(); + + // All 6 columns are "normal" columns and are returned in-place. + reader._normal_columns_idx = {KEY_IDX, VAL_IDX, BEFORE_VAL_IDX, TSO_IDX, LSN_IDX, OP_IDX}; + reader._return_columns_loc = {0, 1, 2, 3, 4, 5}; + + reader._next_row.block = source; + reader._next_row.row_pos = 0; + reader._next_row.is_same = false; + reader._eof = false; + + reader._vcollect_iter._inner_iter = + std::make_unique(&reader, std::move(source)); +} + +// Build the empty output block matching the source schema. +Block make_output_block() { + Block block; + auto type = std::make_shared(); + block.insert({ColumnInt64::create(), type, "key"}); + block.insert({ColumnInt64::create(), type, "val"}); + block.insert({ColumnInt64::create(), type, binlog::build_before_column_name("val")}); + block.insert({ColumnInt64::create(), type, BINLOG_TSO_COL}); + block.insert({ColumnInt64::create(), type, BINLOG_LSN_COL}); + block.insert({ColumnInt64::create(), type, BINLOG_OP_COL}); + return block; +} + +int64_t out_i64(const Block& block, int col, int row) { + return assert_cast(*block.get_by_position(col).column).get_element(row); +} + +struct OutRow { + int64_t key; + int64_t val; + int64_t op; +}; + +// Drain the reader across as many next_block calls as needed and collect the +// (key, val, op) triples in output order. +std::vector drain(BlockReader& reader, Status (BlockReader::*fn)(Block*, bool*)) { + std::vector result; + bool eof = false; + int guard = 0; + while (!eof) { + Block block = make_output_block(); + Status st = (reader.*fn)(&block, &eof); + EXPECT_TRUE(st.ok()) << st; + for (size_t r = 0; r < block.rows(); ++r) { + result.push_back({out_i64(block, KEY_IDX, r), out_i64(block, VAL_IDX, r), + out_i64(block, OP_IDX, r)}); + } + if (++guard >= 1000) { + ADD_FAILURE() << "drain did not terminate"; + break; + } + } + return result; +} + +} // namespace + +class BlockReaderChangeNextBlockTest : public testing::Test { +protected: + void SetUp() override { _saved_adaptive = config::enable_adaptive_batch_size; } + void TearDown() override { config::enable_adaptive_batch_size = _saved_adaptive; } + bool _saved_adaptive = false; +}; + +// ============================================================================ +// _min_delta_next_block branch coverage +// ============================================================================ + +// APPEND then DELETE within the window folds to SKIP: nothing is emitted. +TEST_F(BlockReaderChangeNextBlockTest, MinDeltaSkip) { + auto source = make_source_block({ + {/*key=*/1, /*val=*/10, /*before=*/0, /*tso=*/1, /*lsn=*/1, ROW_BINLOG_APPEND}, + {/*key=*/1, /*val=*/10, /*before=*/10, /*tso=*/2, /*lsn=*/2, ROW_BINLOG_DELETE}, + }); + BlockReader reader; + configure_reader(reader, source, /*batch_size=*/16); + + auto out = drain(reader, &BlockReader::_min_delta_next_block); + EXPECT_TRUE(out.empty()); +} + +// APPEND (+ later UPDATE) folds to a single INSERT carrying the most recent value. +TEST_F(BlockReaderChangeNextBlockTest, MinDeltaInsert) { + auto source = make_source_block({ + {1, 10, 0, 1, 1, ROW_BINLOG_APPEND}, + {1, 20, 10, 2, 2, ROW_BINLOG_UPDATE}, + }); + BlockReader reader; + configure_reader(reader, source, 16); + + auto out = drain(reader, &BlockReader::_min_delta_next_block); + ASSERT_EQ(out.size(), 1); + EXPECT_EQ(out[0].op, binlog::STREAM_CHANGE_INSERT); + EXPECT_EQ(out[0].key, 1); + EXPECT_EQ(out[0].val, 20); // most recent value +} + +// UPDATE then DELETE folds to a single DELETE carrying the first op's before value. +TEST_F(BlockReaderChangeNextBlockTest, MinDeltaDelete) { + auto source = make_source_block({ + {1, 20, 10, 1, 1, ROW_BINLOG_UPDATE}, + {1, 20, 20, 2, 2, ROW_BINLOG_DELETE}, + }); + BlockReader reader; + configure_reader(reader, source, 16); + + auto out = drain(reader, &BlockReader::_min_delta_next_block); + ASSERT_EQ(out.size(), 1); + EXPECT_EQ(out[0].op, binlog::STREAM_CHANGE_DELETE); + EXPECT_EQ(out[0].key, 1); + // delete uses the first op's before value (val's __BEFORE__ mirror of row 0). + EXPECT_EQ(out[0].val, 10); +} + +// UPDATE then UPDATE folds to a BEFORE + AFTER pair. +TEST_F(BlockReaderChangeNextBlockTest, MinDeltaUpdateBeforeAfter) { + auto source = make_source_block({ + {1, 20, 10, 1, 1, ROW_BINLOG_UPDATE}, + {1, 30, 20, 2, 2, ROW_BINLOG_UPDATE}, + }); + BlockReader reader; + configure_reader(reader, source, 16); + + auto out = drain(reader, &BlockReader::_min_delta_next_block); + ASSERT_EQ(out.size(), 2); + EXPECT_EQ(out[0].op, binlog::STREAM_CHANGE_UPDATE_BEFORE); + EXPECT_EQ(out[0].val, 10); // before value from the first op + EXPECT_EQ(out[1].op, binlog::STREAM_CHANGE_UPDATE_AFTER); + EXPECT_EQ(out[1].val, 30); // after value from the last op +} + +// Multiple distinct keys, each in its own group, are folded independently. +TEST_F(BlockReaderChangeNextBlockTest, MinDeltaMultipleKeys) { + auto source = make_source_block({ + // key 1: APPEND -> INSERT(val=10) + {1, 10, 0, 1, 1, ROW_BINLOG_APPEND}, + // key 2: APPEND + DELETE -> SKIP + {2, 20, 0, 2, 2, ROW_BINLOG_APPEND}, + {2, 20, 20, 3, 3, ROW_BINLOG_DELETE}, + // key 3: UPDATE + DELETE -> DELETE(before=30) + {3, 40, 30, 4, 4, ROW_BINLOG_UPDATE}, + {3, 40, 40, 5, 5, ROW_BINLOG_DELETE}, + }); + BlockReader reader; + configure_reader(reader, source, 16); + + auto out = drain(reader, &BlockReader::_min_delta_next_block); + ASSERT_EQ(out.size(), 2); + EXPECT_EQ(out[0].key, 1); + EXPECT_EQ(out[0].op, binlog::STREAM_CHANGE_INSERT); + EXPECT_EQ(out[0].val, 10); + EXPECT_EQ(out[1].key, 3); + EXPECT_EQ(out[1].op, binlog::STREAM_CHANGE_DELETE); + EXPECT_EQ(out[1].val, 30); +} + +// The UPDATE_BEFORE_AFTER split across a batch boundary: BEFORE emitted at the +// end of one block, AFTER carried over via _pending_row_columns to the next. +TEST_F(BlockReaderChangeNextBlockTest, MinDeltaUpdatePendingRowAcrossBatch) { + auto source = make_source_block({ + // key 1: single INSERT fills row 0 of the first batch. + {1, 10, 0, 1, 1, ROW_BINLOG_APPEND}, + // key 2: UPDATE+UPDATE -> BEFORE lands on row 1 (== batch_size), so + // AFTER must be parked and flushed on the next call. + {2, 20, 15, 2, 2, ROW_BINLOG_UPDATE}, + {2, 30, 20, 3, 3, ROW_BINLOG_UPDATE}, + }); + BlockReader reader; + configure_reader(reader, source, /*batch_size=*/2); + + bool eof = false; + Block block1 = make_output_block(); + ASSERT_TRUE(reader._min_delta_next_block(&block1, &eof).ok()); + ASSERT_EQ(block1.rows(), 2); + EXPECT_EQ(out_i64(block1, OP_IDX, 0), binlog::STREAM_CHANGE_INSERT); + EXPECT_EQ(out_i64(block1, OP_IDX, 1), binlog::STREAM_CHANGE_UPDATE_BEFORE); + EXPECT_EQ(out_i64(block1, VAL_IDX, 1), 15); // before value + EXPECT_FALSE(eof); + EXPECT_TRUE(reader._has_pending_row); + + Block block2 = make_output_block(); + ASSERT_TRUE(reader._min_delta_next_block(&block2, &eof).ok()); + ASSERT_EQ(block2.rows(), 1); + EXPECT_EQ(out_i64(block2, OP_IDX, 0), binlog::STREAM_CHANGE_UPDATE_AFTER); + EXPECT_EQ(out_i64(block2, VAL_IDX, 0), 30); // after value + EXPECT_TRUE(eof); +} + +// ============================================================================ +// _detail_change_next_block branch coverage +// ============================================================================ + +// APPEND -> a single INSERT row with the row's value. +TEST_F(BlockReaderChangeNextBlockTest, DetailInsert) { + auto source = make_source_block({ + {1, 10, 0, 1, 1, ROW_BINLOG_APPEND}, + }); + BlockReader reader; + configure_reader(reader, source, 16); + + auto out = drain(reader, &BlockReader::_detail_change_next_block); + ASSERT_EQ(out.size(), 1); + EXPECT_EQ(out[0].op, binlog::STREAM_CHANGE_INSERT); + EXPECT_EQ(out[0].key, 1); + EXPECT_EQ(out[0].val, 10); +} + +// DELETE -> a single DELETE row that uses the before value. +TEST_F(BlockReaderChangeNextBlockTest, DetailDelete) { + auto source = make_source_block({ + {1, 10, 99, 1, 1, ROW_BINLOG_DELETE}, + }); + BlockReader reader; + configure_reader(reader, source, 16); + + auto out = drain(reader, &BlockReader::_detail_change_next_block); + ASSERT_EQ(out.size(), 1); + EXPECT_EQ(out[0].op, binlog::STREAM_CHANGE_DELETE); + EXPECT_EQ(out[0].key, 1); + EXPECT_EQ(out[0].val, 99); // delete uses __BEFORE__ mirror +} + +// UPDATE -> a BEFORE (before value) + AFTER (after value) pair. +TEST_F(BlockReaderChangeNextBlockTest, DetailUpdatePair) { + auto source = make_source_block({ + {1, 20, 10, 1, 1, ROW_BINLOG_UPDATE}, + }); + BlockReader reader; + configure_reader(reader, source, 16); + + auto out = drain(reader, &BlockReader::_detail_change_next_block); + ASSERT_EQ(out.size(), 2); + EXPECT_EQ(out[0].op, binlog::STREAM_CHANGE_UPDATE_BEFORE); + EXPECT_EQ(out[0].val, 10); // before + EXPECT_EQ(out[1].op, binlog::STREAM_CHANGE_UPDATE_AFTER); + EXPECT_EQ(out[1].val, 20); // after +} + +// Mixed ops emitted verbatim in order. +TEST_F(BlockReaderChangeNextBlockTest, DetailMixedOps) { + auto source = make_source_block({ + {1, 10, 0, 1, 1, ROW_BINLOG_APPEND}, + {2, 25, 20, 2, 2, ROW_BINLOG_UPDATE}, + {3, 30, 30, 3, 3, ROW_BINLOG_DELETE}, + }); + BlockReader reader; + configure_reader(reader, source, 16); + + auto out = drain(reader, &BlockReader::_detail_change_next_block); + ASSERT_EQ(out.size(), 4); // INSERT + (BEFORE,AFTER) + DELETE + EXPECT_EQ(out[0].op, binlog::STREAM_CHANGE_INSERT); + EXPECT_EQ(out[0].val, 10); + EXPECT_EQ(out[1].op, binlog::STREAM_CHANGE_UPDATE_BEFORE); + EXPECT_EQ(out[1].val, 20); + EXPECT_EQ(out[2].op, binlog::STREAM_CHANGE_UPDATE_AFTER); + EXPECT_EQ(out[2].val, 25); + EXPECT_EQ(out[3].op, binlog::STREAM_CHANGE_DELETE); + EXPECT_EQ(out[3].val, 30); +} + +// UPDATE whose BEFORE row fills the batch: AFTER is parked in _pending_row_columns +// and flushed at the start of the next call. +TEST_F(BlockReaderChangeNextBlockTest, DetailUpdatePendingRowAcrossBatch) { + auto source = make_source_block({ + {1, 10, 0, 1, 1, ROW_BINLOG_APPEND}, + {2, 25, 20, 2, 2, ROW_BINLOG_UPDATE}, + }); + BlockReader reader; + configure_reader(reader, source, /*batch_size=*/2); + + bool eof = false; + Block block1 = make_output_block(); + ASSERT_TRUE(reader._detail_change_next_block(&block1, &eof).ok()); + ASSERT_EQ(block1.rows(), 2); + EXPECT_EQ(out_i64(block1, OP_IDX, 0), binlog::STREAM_CHANGE_INSERT); + EXPECT_EQ(out_i64(block1, OP_IDX, 1), binlog::STREAM_CHANGE_UPDATE_BEFORE); + EXPECT_EQ(out_i64(block1, VAL_IDX, 1), 20); // before + EXPECT_FALSE(eof); + EXPECT_TRUE(reader._has_pending_row); + + Block block2 = make_output_block(); + ASSERT_TRUE(reader._detail_change_next_block(&block2, &eof).ok()); + ASSERT_EQ(block2.rows(), 1); + EXPECT_EQ(out_i64(block2, OP_IDX, 0), binlog::STREAM_CHANGE_UPDATE_AFTER); + EXPECT_EQ(out_i64(block2, VAL_IDX, 0), 25); // after + EXPECT_TRUE(eof); +} + +// Already-at-EOF with no pending row returns eof immediately with no output. +TEST_F(BlockReaderChangeNextBlockTest, DetailEofImmediately) { + auto source = make_source_block({ + {1, 10, 0, 1, 1, ROW_BINLOG_APPEND}, + }); + BlockReader reader; + configure_reader(reader, source, 16); + + // First call drains the only row. + auto out = drain(reader, &BlockReader::_detail_change_next_block); + ASSERT_EQ(out.size(), 1); + + // A subsequent call should report eof with no rows. + bool eof = false; + Block block = make_output_block(); + ASSERT_TRUE(reader._detail_change_next_block(&block, &eof).ok()); + EXPECT_EQ(block.rows(), 0); + EXPECT_TRUE(eof); +} + +} // namespace doris diff --git a/be/test/storage/olap_type_test.cpp b/be/test/storage/olap_type_test.cpp index 741f75bf47b9ab..3f456b47e682a0 100644 --- a/be/test/storage/olap_type_test.cpp +++ b/be/test/storage/olap_type_test.cpp @@ -240,160 +240,53 @@ TEST_F(OlapTypeTest, deser_double_old) { input_file.close(); } TEST_F(OlapTypeTest, ser_deser_float) { + // to_olap_string uses the shortest round-trippable form; the expected strings + // below are that shortest representation for each value. std::vector> normal_input_values = { - { - 1.230F, - "1.23", // trailing zero of fractional part is not printed - }, - { - 123.456000F, - "123.456", - }, - { - 123.000F, - "123", // decimal point is printed if fractional part is zero - }, - { - 1234567.000F, - "1234567", - }, - // 7 > e >= -4: decimal format. - // e < -4 or e >= 7: scientific format. - { - 123456.12345F, - "123456.1", - }, - { - 1234567.12345F, - "1234567", - }, - { - 12345678.12345F, - "1.234568e+07", - }, - { - 123456789.12345F, - "1.234568e+08", - }, - { - 1234567890000.12345F, - "1.234568e+12", - }, - { - 0.33F, - "0.33", - }, - { - 123.456F, - "123.456", - }, - { - 123.456789F, - "123.4568", - }, - { - 123.456789123F, - "123.4568", - }, - { - 123456.123456789F, - "123456.1", - }, - { - 1234567.123456789F, - "1234567", - }, - { - 987654336.0F, - "9.876543e+08", - }, - { - 16777216.0F, - "1.677722e+07", - }, - { - 0.000123456F, - "0.000123456", - }, - { - 0.0001234567F, - "0.0001234567", - }, - { - 0.00012345678F, - "0.0001234568", - }, - { - 0.0000123456F, - "1.23456e-05", // e < -4 - }, - { - 0.00001234567F, - "1.234567e-05", // e < -4 - }, - { - 0.0000123456789F, - "1.234568e-05", - }, - { - 0.000000000000001234567F, - "1.234567e-15", - }, - { - 0.000000000000001234567890123456F, - "1.234568e-15", - }, - { - 0.1234567F, - "0.1234567", - }, - { - 0.123456789F, - "0.1234568", - }, - - { - 1234567890123456.12345F, - "1.234568e+15", - }, - { - 12345678901234567.12345F, - "1.234568e+16", - }}; - for (int i = 1; i < 10; ++i) { - normal_input_values.emplace_back(i * 0.0000001F, fmt::format("{}e-07", i)); - normal_input_values.emplace_back(i * 0.000000001F, fmt::format("{}e-09", i)); - } + {1.230F, "1.23"}, + {123.456000F, "123.456"}, + {123.000F, "123"}, + {1234567.000F, "1234567"}, + {123456.12345F, "123456.125"}, + {1234567.12345F, "1234567.1"}, + {12345678.12345F, "12345678"}, + {123456789.12345F, "123456790"}, + {1234567890000.12345F, "1234568000000"}, + {0.33F, "0.33"}, + {123.456F, "123.456"}, + {123.456789F, "123.45679"}, + {123.456789123F, "123.45679"}, + {123456.123456789F, "123456.125"}, + {1234567.123456789F, "1234567.1"}, + {987654336.0F, "987654340"}, + {16777216.0F, "16777216"}, + {0.000123456F, "0.000123456"}, + {0.0001234567F, "0.0001234567"}, + {0.00012345678F, "0.00012345678"}, + {0.0000123456F, "1.23456e-05"}, + {0.00001234567F, "1.234567e-05"}, + {0.0000123456789F, "1.2345679e-05"}, + {0.000000000000001234567F, "1.234567e-15"}, + {0.000000000000001234567890123456F, "1.2345679e-15"}, + {0.1234567F, "0.1234567"}, + {0.123456789F, "0.12345679"}, + {1234567890123456.12345F, "1234568000000000"}, + {12345678901234567.12345F, "1.2345678e+16"}}; std::vector> test_input_values; for (const auto& [float_value, expected_str] : normal_input_values) { test_input_values.emplace_back(float_value, expected_str); test_input_values.emplace_back(-float_value, fmt::format("-{}", expected_str)); } std::vector> special_input_values = { - { - 0.0, - "0", - }, - { - -0.0, - "-0", - }, - {std::numeric_limits::min(), "1.175494e-38"}, - {std::numeric_limits::lowest(), "-3.402823e+38"}, - {std::numeric_limits::denorm_min(), "1.401298e-45"}, - {std::numeric_limits::max(), "3.402823e+38"}, - { - std::numeric_limits::infinity(), - "Infinity", - }, - { - -std::numeric_limits::infinity(), - "-Infinity", - }, - { - std::numeric_limits::quiet_NaN(), - "NaN", - }}; + {0.0, "0"}, + {-0.0, "-0"}, + {std::numeric_limits::min(), "1.1754944e-38"}, + {std::numeric_limits::lowest(), "-3.4028235e+38"}, + {std::numeric_limits::denorm_min(), "1e-45"}, + {std::numeric_limits::max(), "3.4028235e+38"}, + {std::numeric_limits::infinity(), "Infinity"}, + {-std::numeric_limits::infinity(), "-Infinity"}, + {std::numeric_limits::quiet_NaN(), "NaN"}}; test_input_values.insert(test_input_values.end(), special_input_values.begin(), special_input_values.end()); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, false); @@ -401,7 +294,8 @@ TEST_F(OlapTypeTest, ser_deser_float) { for (const auto& [float_value, expected_str] : test_input_values) { auto field = Field::create_field(float_value); auto result_str = data_type_serde->to_olap_string(field); - EXPECT_EQ(result_str, expected_str); + EXPECT_EQ(result_str, expected_str) + << "Float to_olap_string mismatch for " << fmt::format("{:.9g}", float_value); Field restored_field; auto status = data_type_serde->from_fe_string(result_str, restored_field); // from_fe_string rejects NaN/Infinity strings @@ -410,193 +304,67 @@ TEST_F(OlapTypeTest, ser_deser_float) { continue; } EXPECT_TRUE(status.ok()) << status.to_string(); - float deser_float_value = restored_field.get(); - float diff_ratio = std::abs(deser_float_value - float_value) / abs(float_value); - EXPECT_TRUE((float_value == 0 && deser_float_value == 0) || diff_ratio < 1e-6) - << "expected float value: " << fmt::format("{:.9g}", float_value) - << ", expected float str: " << expected_str - << ", deser float value: " << fmt::format("{:.9g}", deser_float_value) - << ", diff_ratio: " << fmt::format("{:.9g}", diff_ratio); + // Shortest form round-trips exactly. + EXPECT_EQ(restored_field.get(), float_value) + << "Float round-trip mismatch for '" << result_str << "'"; } } TEST_F(OlapTypeTest, ser_deser_double) { + // to_olap_string uses the shortest round-trippable form; the expected strings + // below are that shortest representation for each value. std::vector> normal_input_values = { - { - 1.230, - "1.23", // trailing zero of fractional part is not printed - }, - { - 123.456000, - "123.456", - }, - { - 123.000, - "123", // decimal point is printed if fractional part is zero - }, - { - 1234567.000, - "1234567", - }, - // 16 > e >= -4: decimal format. - // e < -4 or e >= 16: scientific format. - { - 123456.12345, - "123456.12345", - }, - { - 1234567.12345, - "1234567.12345", - }, - { - 12345678.12345, - "12345678.12345", - }, - { - 123456789.12345, - "123456789.12345", - }, - { - 1234567890000.12345, - "1234567890000.124", - }, - { - 0.33, - "0.33", - }, - { - 123.456, - "123.456", - }, - { - 123.456789, - "123.456789", - }, - { - 123.456789123, - "123.456789123", - }, - { - 123456.123456789, - "123456.123456789", - }, - { - 1234567.123456789, - "1234567.123456789", - }, - { - 987654336.0, - "987654336", - }, - { - 16777216.0, - "16777216", - }, - { - 0.000123456, - "0.000123456", - }, - { - 0.0001234567, - "0.0001234567", - }, - { - 0.00012345678, - "0.00012345678", - }, - { - 0.0000123456, - "1.23456e-05", // e < -4 - }, - { - 0.00001234567, - "1.234567e-05", // e < -4 - }, - { - 0.0000123456789, - "1.23456789e-05", - }, - { - 0.000000000000001234567, - "1.234567e-15", - }, - { - 0.000000000000001234567890123456, - "1.234567890123456e-15", - }, - { - 0.1234567, - "0.1234567", - }, - { - 0.123456789, - "0.123456789", - }, - - { - 1234567890123456.12345, - "1234567890123456", - }, - { - 12345678901234567.12345, - "1.234567890123457e+16", - }, - { - 123456789012345678.12345, - "1.234567890123457e+17", - }}; - for (int i = 1; i < 10; ++i) { - if (i == 7) { - normal_input_values.emplace_back(i * 0.0000000000000001, "6.999999999999999e-16"); - continue; - } - normal_input_values.emplace_back(i * 0.0000000000000001, fmt::format("{}e-16", i)); - } + {1.230, "1.23"}, + {123.456000, "123.456"}, + {123.000, "123"}, + {1234567.000, "1234567"}, + {123456.12345, "123456.12345"}, + {1234567.12345, "1234567.12345"}, + {12345678.12345, "12345678.12345"}, + {123456789.12345, "123456789.12345"}, + {1234567890000.12345, "1234567890000.1235"}, + {0.33, "0.33"}, + {123.456, "123.456"}, + {123.456789, "123.456789"}, + {123.456789123, "123.456789123"}, + {123456.123456789, "123456.123456789"}, + {1234567.123456789, "1234567.123456789"}, + {987654336.0, "987654336"}, + {16777216.0, "16777216"}, + {0.000123456, "0.000123456"}, + {0.0001234567, "0.0001234567"}, + {0.00012345678, "0.00012345678"}, + {0.0000123456, "1.23456e-05"}, + {0.00001234567, "1.234567e-05"}, + {0.0000123456789, "1.23456789e-05"}, + {0.000000000000001234567, "1.234567e-15"}, + {0.000000000000001234567890123456, "1.234567890123456e-15"}, + {0.1234567, "0.1234567"}, + {0.123456789, "0.123456789"}, + {1234567890123456.12345, "1234567890123456"}, + {12345678901234567.12345, "1.2345678901234568e+16"}, + {123456789012345678.12345, "1.2345678901234568e+17"}}; std::vector> test_input_values; for (const auto& [float_value, expected_str] : normal_input_values) { test_input_values.emplace_back(float_value, expected_str); test_input_values.emplace_back(-float_value, fmt::format("-{}", expected_str)); } std::vector> special_input_values = { - { - 0.0, - "0", - }, - { - -0.0, - "-0", - }, - {std::numeric_limits::min(), "1.175494350822288e-38"}, - {std::numeric_limits::lowest(), "-3.402823466385289e+38"}, + {0.0, "0"}, + {-0.0, "-0"}, + {std::numeric_limits::min(), "1.1754943508222875e-38"}, + {std::numeric_limits::lowest(), "-3.4028234663852886e+38"}, {std::numeric_limits::denorm_min(), "1.401298464324817e-45"}, - {std::numeric_limits::max(), "3.402823466385289e+38"}, - {std::numeric_limits::min(), "2.225073858507201e-308"}, - {std::numeric_limits::lowest(), "-1.797693134862316e+308"}, - {std::numeric_limits::denorm_min(), "4.940656458412465e-324"}, - {std::numeric_limits::max(), "1.797693134862316e+308"}, - { - std::numeric_limits::infinity(), - "Infinity", - }, - { - -std::numeric_limits::infinity(), - "-Infinity", - }, - { - std::numeric_limits::quiet_NaN(), - "NaN", - }, - { - std::numeric_limits::infinity(), - "Infinity", - }, - { - -std::numeric_limits::infinity(), - "-Infinity", - }, - { - std::numeric_limits::quiet_NaN(), - "NaN", - }}; + {std::numeric_limits::max(), "3.4028234663852886e+38"}, + {std::numeric_limits::min(), "2.2250738585072014e-308"}, + {std::numeric_limits::lowest(), "-1.7976931348623157e+308"}, + {std::numeric_limits::denorm_min(), "5e-324"}, + {std::numeric_limits::max(), "1.7976931348623157e+308"}, + {std::numeric_limits::infinity(), "Infinity"}, + {-std::numeric_limits::infinity(), "-Infinity"}, + {std::numeric_limits::quiet_NaN(), "NaN"}, + {std::numeric_limits::infinity(), "Infinity"}, + {-std::numeric_limits::infinity(), "-Infinity"}, + {std::numeric_limits::quiet_NaN(), "NaN"}}; test_input_values.insert(test_input_values.end(), special_input_values.begin(), special_input_values.end()); auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, false); @@ -604,25 +372,19 @@ TEST_F(OlapTypeTest, ser_deser_double) { for (const auto& [float_value, expected_str] : test_input_values) { auto field = Field::create_field(float_value); auto result_str = data_type_serde->to_olap_string(field); - EXPECT_EQ(result_str, expected_str); + EXPECT_EQ(result_str, expected_str) + << "Double to_olap_string mismatch for " << fmt::format("{:.17g}", float_value); Field restored_field; auto status = data_type_serde->from_fe_string(result_str, restored_field); - // from_fe_string rejects NaN/Infinity strings, and also rejects - // double::max()/lowest() whose string representation parses to Infinity - if (std::isnan(float_value) || std::isinf(float_value) || - float_value == std::numeric_limits::max() || - float_value == std::numeric_limits::lowest()) { + // from_fe_string rejects NaN/Infinity strings + if (std::isnan(float_value) || std::isinf(float_value)) { EXPECT_FALSE(status.ok()); continue; } EXPECT_TRUE(status.ok()) << status.to_string(); - double deser_float_value = restored_field.get(); - double diff_ratio = std::abs(deser_float_value - float_value) / abs(float_value); - EXPECT_TRUE((float_value == 0 && deser_float_value == 0) || diff_ratio < 1e-15) - << "expected double value: " << fmt::format("{:.17g}", float_value) - << ", expected double str: " << expected_str - << ", deser double value: " << fmt::format("{:.17g}", deser_float_value) - << ", diff_ratio: " << fmt::format("{:.17g}", diff_ratio); + // Shortest form round-trips exactly. + EXPECT_EQ(restored_field.get(), float_value) + << "Double round-trip mismatch for '" << result_str << "'"; } } @@ -973,8 +735,7 @@ TEST_F(OlapTypeTest, ser_deser_float_olap_string) { auto status = serde->from_zonemap_string(result_str, restored_field); EXPECT_TRUE(status.ok()) << status.to_string(); float restored_val = restored_field.get(); - float diff = std::abs(restored_val - val); - EXPECT_TRUE(val == 0 ? restored_val == 0 : diff / std::abs(val) < 1e-6) + EXPECT_EQ(restored_val, val) << "Float round-trip: expected " << val << ", got " << restored_val; } @@ -1007,12 +768,9 @@ TEST_F(OlapTypeTest, ser_deser_float_olap_string) { } // --------------------------------------------------------------------------- -// Double: same pattern as Float. -// The expected strings in this case follow current serializer behavior. -// Note: for DBL_MAX/lowest, current formatting rounds to a boundary string that -// is rejected by from_zonemap_string (parsed as Infinity), so these two values -// are validated for to_olap_string only. -// NaN/Inf same behavior: to_olap_string works, from_zonemap_string rejects. +// Double: same pattern as Float. The shortest form round-trips every finite +// value, including DBL_MAX/lowest. NaN/Inf are tracked via zone-map flags, so +// to_olap_string emits them but from_zonemap_string rejects the string. // --------------------------------------------------------------------------- TEST_F(OlapTypeTest, ser_deser_double_olap_string) { auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, false); @@ -1026,8 +784,8 @@ TEST_F(OlapTypeTest, ser_deser_double_olap_string) { {0.001, "0.001"}, {1234567890123456.0, "1234567890123456"}, {1e-100, "1e-100"}, - {std::numeric_limits::lowest(), "-1.797693134862316e+308"}, - {std::numeric_limits::max(), "1.797693134862316e+308"}, + {std::numeric_limits::lowest(), "-1.7976931348623157e+308"}, + {std::numeric_limits::max(), "1.7976931348623157e+308"}, }; for (const auto& [val, expected_str] : normal_cases) { @@ -1039,19 +797,9 @@ TEST_F(OlapTypeTest, ser_deser_double_olap_string) { // Round-trip Field restored_field; auto status = serde->from_zonemap_string(result_str, restored_field); - if (val == std::numeric_limits::lowest() || - val == std::numeric_limits::max()) { - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("NaN/Infinity not allowed in olap string"), - std::string::npos) - << status.to_string(); - continue; - } - EXPECT_TRUE(status.ok()) << status.to_string(); double restored_val = restored_field.get(); - double diff = std::abs(restored_val - val); - EXPECT_TRUE(val == 0 ? restored_val == 0 : diff / std::abs(val) < 1e-15) + EXPECT_EQ(restored_val, val) << "Double round-trip: expected " << val << ", got " << restored_val; } @@ -1794,18 +1542,31 @@ TEST_F(OlapTypeTest, float_type) { {std::numeric_limits::quiet_NaN(), "NaN", "NaN"}, {std::numeric_limits::infinity(), "Infinity", "Infinity"}, {-std::numeric_limits::infinity(), "-Infinity", "-Infinity"}, - {std::numeric_limits::max(), "3.402823e+38", "3.402823e+38"}, - {std::numeric_limits::lowest(), "-3.402823e+38", "-3.402823e+38"}, - {std::numeric_limits::min(), "1.175494e-38", "1.175494e-38"}, - {std::numeric_limits::denorm_min(), "1.401298e-45", "1.401298e-45"}, + {std::numeric_limits::max(), "3.4028235e+38", "3.4028235e+38"}, + {std::numeric_limits::lowest(), "-3.4028235e+38", "-3.4028235e+38"}, + {std::numeric_limits::min(), "1.1754944e-38", "1.1754944e-38"}, + {std::numeric_limits::denorm_min(), "1e-45", "1e-45"}, }; for (auto& tc : test_cases) { auto field = Field::create_field(tc.value); std::string serde_str = serde->to_olap_string(field); - EXPECT_EQ(tc.expected_serde, serde_str) - << "serde mismatch for FLOAT expected='" << tc.expected << "'"; + // to_olap_string uses the shortest round-trippable form. + EXPECT_EQ(tc.expected_serde, serde_str) << "serde mismatch for FLOAT=" << tc.expected; + + // It must also round-trip back to the same value via from_zonemap_string. + Field rf; + auto st = serde->from_zonemap_string(serde_str, rf); + if (std::isfinite(tc.value)) { + EXPECT_TRUE(st.ok()) << "FLOAT to_olap_string='" << serde_str + << "': " << st.to_string(); + EXPECT_EQ(rf.get(), tc.value) + << "FLOAT round-trip mismatch, serde='" << serde_str << "'"; + } else { + // NaN/Inf are tracked via zone-map flags; from_zonemap_string rejects the string. + EXPECT_FALSE(st.ok()) << "FLOAT to_olap_string='" << serde_str << "'"; + } } } @@ -1832,20 +1593,33 @@ TEST_F(OlapTypeTest, double_type) { {std::numeric_limits::quiet_NaN(), "NaN", "NaN"}, {std::numeric_limits::infinity(), "Infinity", "Infinity"}, {-std::numeric_limits::infinity(), "-Infinity", "-Infinity"}, - {std::numeric_limits::max(), "1.797693134862316e+308", - "1.797693134862316e+308"}, - {std::numeric_limits::lowest(), "-1.797693134862316e+308", - "-1.797693134862316e+308"}, - {std::numeric_limits::min(), "2.225073858507201e-308", - "2.225073858507201e-308"}, + {std::numeric_limits::max(), "1.7976931348623157e+308", + "1.7976931348623157e+308"}, + {std::numeric_limits::lowest(), "-1.7976931348623157e+308", + "-1.7976931348623157e+308"}, + {std::numeric_limits::min(), "2.2250738585072014e-308", + "2.2250738585072014e-308"}, }; for (auto& tc : test_cases) { auto field = Field::create_field(tc.value); std::string serde_str = serde->to_olap_string(field); - EXPECT_EQ(tc.expected_serde, serde_str) - << "serde mismatch for DOUBLE expected='" << tc.expected << "'"; + // to_olap_string uses the shortest round-trippable form. + EXPECT_EQ(tc.expected_serde, serde_str) << "serde mismatch for DOUBLE=" << tc.expected; + + // It must also round-trip back to the same value via from_zonemap_string. + Field rf; + auto st = serde->from_zonemap_string(serde_str, rf); + if (std::isfinite(tc.value)) { + EXPECT_TRUE(st.ok()) << "DOUBLE to_olap_string='" << serde_str + << "': " << st.to_string(); + EXPECT_EQ(rf.get(), tc.value) + << "DOUBLE round-trip mismatch, serde='" << serde_str << "'"; + } else { + // NaN/Inf are tracked via zone-map flags; from_zonemap_string rejects the string. + EXPECT_FALSE(st.ok()) << "DOUBLE to_olap_string='" << serde_str << "'"; + } } } diff --git a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp b/be/test/storage/segment/adaptive_block_size_predictor_test.cpp index 60b6f37b8ceeba..64795dace19a14 100644 --- a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp +++ b/be/test/storage/segment/adaptive_block_size_predictor_test.cpp @@ -89,6 +89,18 @@ TEST_F(AdaptiveBlockSizePredictorTest, NoHistoryReturnsMaxRows) { EXPECT_DOUBLE_EQ(pred.bytes_per_row_for_test(), expected_bpr); } +TEST_F(AdaptiveBlockSizePredictorTest, ExplicitMaterializedSampleUsesPreFilterShape) { + AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0); + + // Callers that filter a block before returning it can still report the rows and bytes that + // were actually materialized upstream. + pred.update(32, 32 * 4096); + + EXPECT_TRUE(pred.has_history_for_test()); + EXPECT_DOUBLE_EQ(pred.bytes_per_row_for_test(), 4096.0); + EXPECT_EQ(pred.predict_next_rows(), 2048); +} + // ── Test 2: EWMA convergence ────────────────────────────────────────────────── // When every update delivers the same sample, the EWMA stays exactly at that // value (0.9*v + 0.1*v == v for any v). diff --git a/be/test/storage/segment/column_reader_test.cpp b/be/test/storage/segment/column_reader_test.cpp index 8dfe640808e4fc..3dfe42ac7cd4d2 100644 --- a/be/test/storage/segment/column_reader_test.cpp +++ b/be/test/storage/segment/column_reader_test.cpp @@ -629,6 +629,126 @@ TEST_F(ColumnReaderTest, PlaceHolderRecoveryAfterColumnReplacement) { EXPECT_EQ(2, dst->size()); } +namespace { +void check_default_value_lazy_output(bool read_by_rowids) { + SCOPED_TRACE(read_by_rowids ? "read_by_rowids" : "next_batch"); + + DefaultValueColumnIterator iterator(true, "7", false, FieldType::OLAP_FIELD_TYPE_INT, 0, 0, + sizeof(int32_t)); + ColumnIteratorOptions iter_opts; + ASSERT_TRUE(iterator.init(iter_opts).ok()); + iterator.set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + MutableColumnPtr dst = ColumnInt32::create(); + size_t rows = 3; + bool has_null = false; + ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok()); + EXPECT_TRUE(iterator._has_place_holder_column); + + IColumn::Filter filter; + filter.resize(3); + filter[0] = 1; + filter[1] = 0; + filter[2] = 1; + dst = IColumn::mutate(dst->filter(filter, 2)); + ASSERT_EQ(2, dst->size()); + + iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + if (read_by_rowids) { + const rowid_t rowids[] = {2, 8}; + ASSERT_TRUE(iterator.read_by_rowids(rowids, std::size(rowids), dst).ok()); + } else { + rows = 2; + ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok()); + } + + ASSERT_EQ(2, dst->size()); + EXPECT_FALSE(iterator._has_place_holder_column); + const auto& result = assert_cast(*dst); + EXPECT_EQ(7, result.get_element(0)); + EXPECT_EQ(7, result.get_element(1)); +} + +void check_default_value_predicate_not_read_again(bool read_by_rowids) { + SCOPED_TRACE(read_by_rowids ? "read_by_rowids" : "next_batch"); + + DefaultValueColumnIterator iterator(true, "7", false, FieldType::OLAP_FIELD_TYPE_INT, 0, 0, + sizeof(int32_t)); + ColumnIteratorOptions iter_opts; + ASSERT_TRUE(iterator.init(iter_opts).ok()); + iterator.set_read_requirement(ColumnIterator::ReadRequirement::PREDICATE); + iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + MutableColumnPtr dst = ColumnInt32::create(); + size_t rows = 3; + bool has_null = true; + ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok()); + EXPECT_FALSE(has_null); + EXPECT_FALSE(iterator._has_place_holder_column); + + IColumn::Filter filter; + filter.resize(3); + filter[0] = 1; + filter[1] = 0; + filter[2] = 1; + dst = IColumn::mutate(dst->filter(filter, 2)); + ASSERT_EQ(2, dst->size()); + + iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + if (read_by_rowids) { + const rowid_t rowids[] = {2, 8}; + ASSERT_TRUE(iterator.read_by_rowids(rowids, std::size(rowids), dst).ok()); + } else { + rows = 2; + ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok()); + } + + ASSERT_EQ(2, dst->size()); + const auto& result = assert_cast(*dst); + EXPECT_EQ(7, result.get_element(0)); + EXPECT_EQ(7, result.get_element(1)); +} +} // namespace + +TEST_F(ColumnReaderTest, DefaultValueLazyOutputRecoversFilteredPlaceholder) { + check_default_value_lazy_output(false); + check_default_value_lazy_output(true); +} + +TEST_F(ColumnReaderTest, DefaultValueLazyOutputFinalizesEmptySelection) { + DefaultValueColumnIterator iterator(true, "7", false, FieldType::OLAP_FIELD_TYPE_INT, 0, 0, + sizeof(int32_t)); + ColumnIteratorOptions iter_opts; + ASSERT_TRUE(iterator.init(iter_opts).ok()); + iterator.set_read_requirement(ColumnIterator::ReadRequirement::LAZY_OUTPUT); + iterator.set_read_phase(ColumnIterator::ReadPhase::PREDICATE); + + MutableColumnPtr dst = ColumnInt32::create(); + size_t rows = 3; + bool has_null = false; + ASSERT_TRUE(iterator.next_batch(&rows, dst, &has_null).ok()); + EXPECT_TRUE(iterator._has_place_holder_column); + + IColumn::Filter filter; + filter.resize(3); + filter[0] = 0; + filter[1] = 0; + filter[2] = 0; + dst = IColumn::mutate(dst->filter(filter, 0)); + ASSERT_EQ(0, dst->size()); + + iterator.set_read_phase(ColumnIterator::ReadPhase::LAZY); + iterator.finalize_lazy_phase(dst); + EXPECT_EQ(0, dst->size()); + EXPECT_FALSE(iterator._has_place_holder_column); +} + +TEST_F(ColumnReaderTest, DefaultValuePredicateIsNotReadAgainInLazyPhase) { + check_default_value_predicate_not_read_again(false); + check_default_value_predicate_not_read_again(true); +} + TEST_F(ColumnReaderTest, SetReadRequirementPropagatesToNestedIterators) { auto null_iter = std::make_unique(std::make_shared()); std::vector struct_sub_iters; diff --git a/be/test/storage/segment/inverted_index_reader_test.cpp b/be/test/storage/segment/inverted_index_reader_test.cpp index 982d2c3f14155f..9004e7f1476984 100644 --- a/be/test/storage/segment/inverted_index_reader_test.cpp +++ b/be/test/storage/segment/inverted_index_reader_test.cpp @@ -2721,8 +2721,11 @@ class InvertedIndexReaderTest : public testing::Test { EXPECT_TRUE(status.ok()) << status; for (const auto& value : values) { - status = column_writer->add_values(column.name(), reinterpret_cast(&value), - 1); + // Copy into a real element first: for std::vector, `value` is a + // proxy, so `auto`/`&value` would give a __bit_reference/__bit_iterator + // rather than a real pointer. Use the container's value_type. + typename std::decay_t::value_type v = value; + status = column_writer->add_values(column.name(), reinterpret_cast(&v), 1); EXPECT_TRUE(status.ok()) << status; } @@ -3500,8 +3503,11 @@ class InvertedIndexReaderTest : public testing::Test { EXPECT_TRUE(status.ok()) << status; for (const auto& value : values) { - status = column_writer->add_values(column.name(), reinterpret_cast(&value), - 1); + // Copy into a real element first: for std::vector, `value` is a + // proxy, so `auto`/`&value` would give a __bit_reference/__bit_iterator + // rather than a real pointer. Use the container's value_type. + typename std::decay_t::value_type v = value; + status = column_writer->add_values(column.name(), reinterpret_cast(&v), 1); EXPECT_TRUE(status.ok()) << status; } diff --git a/be/test/storage/segment/row_binlog_segment_writer_test.cpp b/be/test/storage/segment/row_binlog_segment_writer_test.cpp new file mode 100644 index 00000000000000..0c0c4b9972f591 --- /dev/null +++ b/be/test/storage/segment/row_binlog_segment_writer_test.cpp @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/segment/row_binlog_segment_writer.h" + +#include + +#include +#include +#include + +#include "core/field.h" +#include "core/value/decimalv2_value.h" +#include "storage/binlog.h" +#include "storage/iterator/olap_data_convertor.h" +#include "storage/tablet/tablet_schema.h" + +namespace doris::segment_v2 { + +namespace { + +TabletColumn create_column(int32_t unique_id, const std::string& name, FieldType type, bool is_key, + bool visible) { + TabletColumn column(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, type, !is_key); + column.set_unique_id(unique_id); + column.set_name(name); + column.set_is_key(is_key); + column.set_length(type == FieldType::OLAP_FIELD_TYPE_LARGEINT ? 16 : 4); + column.set_index_length(column.length()); + column._visible = visible; + return column; +} + +TabletSchemaSPtr create_source_schema() { + auto schema = std::make_shared(); + schema->append_column(create_column(0, "k1", FieldType::OLAP_FIELD_TYPE_INT, true, true)); + schema->append_column(create_column(1, "__DORIS_TEST_HIDDEN_KEY__", + FieldType::OLAP_FIELD_TYPE_LARGEINT, true, false)); + schema->append_column(create_column(2, "v1", FieldType::OLAP_FIELD_TYPE_INT, false, true)); + schema->append_column(create_column(3, "__DORIS_TEST_HIDDEN_VALUE__", + FieldType::OLAP_FIELD_TYPE_INT, false, false)); + schema->_keys_type = UNIQUE_KEYS; + return schema; +} + +Block create_source_block(const TabletSchemaSPtr& schema) { + Block block = schema->create_block(); + auto columns_guard = block.mutate_columns_scoped(); + auto& columns = columns_guard.mutable_columns(); + for (int i = 0; i < 2; ++i) { + columns[0]->insert(Field::create_field(10 + i)); + columns[1]->insert(Field::create_field(static_cast(1000 + i))); + columns[2]->insert(Field::create_field(100 + i)); + columns[3]->insert(Field::create_field(10000 + i)); + } + return block; +} + +} // namespace + +TEST(RowBinlogSegmentWriterTest, collectHiddenKeyInSourceDataWriter) { + auto source_schema = create_source_schema(); + SegmentWriteBinlogOptions options; + options.source.tablet_schema = source_schema; + + RowBinlogSourceDataWriter writer(options); + ASSERT_TRUE(writer.init().ok()); + EXPECT_EQ(3, writer.normal_column_count()); + + Block block = create_source_block(source_schema); + Block full_block = source_schema->create_block(); + std::vector partial_source_cids; + ASSERT_TRUE( + writer.prepare_by_source_block(&block, 0, 2, partial_source_cids, &full_block).ok()); + + const auto& key_columns = writer.source_key_columns(); + ASSERT_EQ(2, key_columns.size()); + + ASSERT_NE(nullptr, key_columns[0]->get_data_at(1)); + EXPECT_EQ(11, *reinterpret_cast(key_columns[0]->get_data_at(1))); + + ASSERT_NE(nullptr, key_columns[1]->get_data_at(1)); + EXPECT_EQ(static_cast(1001), + *reinterpret_cast(key_columns[1]->get_data_at(1))); +} + +TEST(RowBinlogSegmentWriterTest, skipHiddenNonKeyBeforeVisibleColumn) { + auto source_schema = std::make_shared(); + source_schema->append_column( + create_column(0, "k1", FieldType::OLAP_FIELD_TYPE_INT, true, true)); + source_schema->append_column(create_column(1, "__DORIS_TEST_HIDDEN_VALUE__", + FieldType::OLAP_FIELD_TYPE_INT, false, false)); + source_schema->append_column( + create_column(2, "v1", FieldType::OLAP_FIELD_TYPE_INT, false, true)); + source_schema->_keys_type = UNIQUE_KEYS; + + SegmentWriteBinlogOptions options; + options.source.tablet_schema = source_schema; + + RowBinlogSourceDataWriter writer(options); + ASSERT_TRUE(writer.init().ok()); + EXPECT_EQ(2, writer.normal_column_count()); + EXPECT_TRUE(writer.is_normal_column(0)); + EXPECT_FALSE(writer.is_normal_column(1)); + EXPECT_TRUE(writer.is_normal_column(2)); + EXPECT_EQ(0, writer.normal_column_ordinal(0)); + EXPECT_EQ(1, writer.normal_column_ordinal(2)); +} + +TEST(RowBinlogSegmentWriterTest, skipMultipleHiddenNonKeyColumns) { + auto source_schema = std::make_shared(); + source_schema->append_column( + create_column(0, "k1", FieldType::OLAP_FIELD_TYPE_INT, true, true)); + source_schema->append_column(create_column(1, "__DORIS_TEST_HIDDEN_VALUE_1__", + FieldType::OLAP_FIELD_TYPE_INT, false, false)); + source_schema->append_column( + create_column(2, "v1", FieldType::OLAP_FIELD_TYPE_INT, false, true)); + source_schema->append_column(create_column(3, "__DORIS_TEST_HIDDEN_VALUE_2__", + FieldType::OLAP_FIELD_TYPE_INT, false, false)); + source_schema->append_column( + create_column(4, "v2", FieldType::OLAP_FIELD_TYPE_INT, false, true)); + source_schema->append_column(create_column(5, "__DORIS_TEST_HIDDEN_VALUE_3__", + FieldType::OLAP_FIELD_TYPE_INT, false, false)); + source_schema->_keys_type = UNIQUE_KEYS; + + SegmentWriteBinlogOptions options; + options.source.tablet_schema = source_schema; + + RowBinlogSourceDataWriter writer(options); + ASSERT_TRUE(writer.init().ok()); + EXPECT_EQ(3, writer.normal_column_count()); + EXPECT_TRUE(writer.is_normal_column(0)); + EXPECT_FALSE(writer.is_normal_column(1)); + EXPECT_TRUE(writer.is_normal_column(2)); + EXPECT_FALSE(writer.is_normal_column(3)); + EXPECT_TRUE(writer.is_normal_column(4)); + EXPECT_FALSE(writer.is_normal_column(5)); + EXPECT_EQ(0, writer.normal_column_ordinal(0)); + EXPECT_EQ(1, writer.normal_column_ordinal(2)); + EXPECT_EQ(2, writer.normal_column_ordinal(4)); +} + +} // namespace doris::segment_v2 diff --git a/be/test/storage/segment/zone_map_index_test.cpp b/be/test/storage/segment/zone_map_index_test.cpp index 51f8c068947c44..db4b692b1f1477 100644 --- a/be/test/storage/segment/zone_map_index_test.cpp +++ b/be/test/storage/segment/zone_map_index_test.cpp @@ -762,9 +762,8 @@ TEST_F(ColumnZoneMapTest, NormalTestFloatPage) { EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); auto segment_zone_map = index_meta.zone_map_index().segment_zone_map(); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::lowest()), - segment_zone_map.min()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::max()), segment_zone_map.max()); + EXPECT_EQ("-3.4028235e+38", segment_zone_map.min()); + EXPECT_EQ("3.4028235e+38", segment_zone_map.max()); EXPECT_EQ(true, segment_zone_map.has_null()); EXPECT_EQ(true, segment_zone_map.has_not_null()); EXPECT_EQ(true, segment_zone_map.has_positive_inf()); @@ -778,8 +777,8 @@ TEST_F(ColumnZoneMapTest, NormalTestFloatPage) { const std::vector& zone_maps = column_zone_map.page_zone_maps(); EXPECT_EQ(3, zone_maps.size()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::lowest()), zone_maps[0].min()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::max()), zone_maps[0].max()); + EXPECT_EQ("-3.4028235e+38", zone_maps[0].min()); + EXPECT_EQ("3.4028235e+38", zone_maps[0].max()); EXPECT_EQ(false, zone_maps[0].has_null()); EXPECT_EQ(true, zone_maps[0].has_not_null()); EXPECT_EQ(true, zone_maps[0].has_positive_inf()); @@ -851,10 +850,8 @@ TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); auto segment_zone_map = index_meta.zone_map_index().segment_zone_map(); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::lowest()), - segment_zone_map.min()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::max()), - segment_zone_map.max()); + EXPECT_EQ("-1.7976931348623157e+308", segment_zone_map.min()); + EXPECT_EQ("1.7976931348623157e+308", segment_zone_map.max()); EXPECT_EQ(true, segment_zone_map.has_null()); EXPECT_EQ(true, segment_zone_map.has_not_null()); EXPECT_EQ(true, segment_zone_map.has_positive_inf()); @@ -868,8 +865,8 @@ TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { const std::vector& zone_maps = column_zone_map.page_zone_maps(); EXPECT_EQ(3, zone_maps.size()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::lowest()), zone_maps[0].min()); - EXPECT_EQ(CastToString::from_number(std::numeric_limits::max()), zone_maps[0].max()); + EXPECT_EQ("-1.7976931348623157e+308", zone_maps[0].min()); + EXPECT_EQ("1.7976931348623157e+308", zone_maps[0].max()); EXPECT_EQ(false, zone_maps[0].has_null()); EXPECT_EQ(true, zone_maps[0].has_not_null()); EXPECT_EQ(true, zone_maps[0].has_positive_inf()); @@ -885,6 +882,67 @@ TEST_F(ColumnZoneMapTest, NormalTestDoublePage) { EXPECT_EQ(false, zone_maps[2].has_not_null()); } +// A DOUBLE page whose min/max are exactly ±DBL_MAX (with NO ±inf/NaN in the +// page) must round-trip through to_proto/from_proto. When min/max formatting +// used digits10+1 (16g), DBL_MAX serialized as "1.797693134862316e+308" — larger +// than the largest finite double — so from_olap_string parsed it to ±inf, +// rejected it, and ZoneMap::from_proto returned non-OK, leaving the segment +// without a usable zone map (field_types_compatible CHECK / silent prune). +TEST_F(ColumnZoneMapTest, DoubleFiniteExtremesRoundTrip) { + auto column = create_float_column(0, true); + const TabletColumn* field = &(*column); + auto data_type_ptr = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, false); + + std::unique_ptr builder; + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type_ptr, field, builder).ok()); + // Page holds ONLY finite values (incl. ±DBL_MAX) — no ±inf/NaN — so the + // has_*_inf / has_nan flags stay false and from_proto must rebuild min/max + // by parsing the stored strings (exactly the path that broke). + double values[] = { + std::numeric_limits::lowest(), // -DBL_MAX + std::numeric_limits::max(), // +DBL_MAX + -1e308, + 0.0, + 3.141592653589793, + }; + for (double v : values) { + builder->add_values(reinterpret_cast(&v), 1); + } + ASSERT_TRUE(builder->flush().ok()); + + std::string file_path = kTestDir + "/double_finite_extremes"; + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(file_path, &file_writer).ok()); + ColumnIndexMetaPB index_meta; + ASSERT_TRUE(builder->finish(file_writer.get(), &index_meta).ok()); + ASSERT_TRUE(file_writer->close().ok()); + + const auto& seg_zm = index_meta.zone_map_index().segment_zone_map(); + EXPECT_TRUE(seg_zm.has_not_null()); + EXPECT_FALSE(seg_zm.has_positive_inf()); + EXPECT_FALSE(seg_zm.has_negative_inf()); + EXPECT_FALSE(seg_zm.has_nan()); + + ZoneMap zm; + ASSERT_TRUE(ZoneMap::from_proto(seg_zm, data_type_ptr, zm).ok()) + << "±DBL_MAX zone map failed to deserialize"; + EXPECT_EQ(zm.min_value.get(), std::numeric_limits::lowest()); + EXPECT_EQ(zm.max_value.get(), std::numeric_limits::max()); + EXPECT_FALSE(zm.has_positive_inf); + EXPECT_FALSE(zm.has_negative_inf); + EXPECT_FALSE(zm.has_nan); + + io::FileReaderSPtr file_reader; + ASSERT_TRUE(_fs->open_file(file_path, &file_reader).ok()); + ZoneMapIndexReader reader(file_reader, index_meta.zone_map_index().page_zone_maps()); + ASSERT_TRUE(reader.load(true, false).ok()); + ASSERT_EQ(reader.num_pages(), 1); + ZoneMap pzm; + ASSERT_TRUE(ZoneMap::from_proto(reader.page_zone_maps()[0], data_type_ptr, pzm).ok()); + EXPECT_EQ(pzm.min_value.get(), std::numeric_limits::lowest()); + EXPECT_EQ(pzm.max_value.get(), std::numeric_limits::max()); +} + TabletColumnPtr create_timestamptz_column(int32_t id, bool is_nullable) { auto column = std::make_shared(); column->_unique_id = id; diff --git a/be/test/udf/python/python_env_test.cpp b/be/test/udf/python/python_env_test.cpp index 0ccbb63cf1d75e..361d3c59abc87b 100644 --- a/be/test/udf/python/python_env_test.cpp +++ b/be/test/udf/python/python_env_test.cpp @@ -53,7 +53,9 @@ class PythonEnvTest : public ::testing::Test { // which causes pclose() to get ECHILD because the kernel auto-reaps children. // We reset SIGCHLD to SIG_DFL for the duration of each test to mimic production // behaviour, and restore the original handler afterwards. - sighandler_t old_sigchld_ = SIG_DFL; + // sighandler_t is a glibc-only typedef; use a plain function-pointer type so + // this also builds on macOS (where signal() uses sig_t). + void (*old_sigchld_)(int) = SIG_DFL; void SetUp() override { test_dir_ = fs::temp_directory_path().string() + "/python_env_test_" + diff --git a/be/test/util/bit_packing_test.cpp b/be/test/util/bit_packing_test.cpp new file mode 100644 index 00000000000000..26d6708b63e6c8 --- /dev/null +++ b/be/test/util/bit_packing_test.cpp @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include "util/bit_stream_utils.inline.h" +#include "util/faststring.h" + +namespace doris { +namespace { + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +static_assert(PdepUnpack::should_use()); +static_assert(PdepUnpack::should_use()); +static_assert(!PdepUnpack::should_use()); +static_assert(!PdepUnpack::should_use()); +static_assert(!PdepUnpack::should_use()); +#endif + +template +std::vector make_values(int bit_width, int num_values) { + const uint64_t mask = bit_width == std::numeric_limits::digits + ? std::numeric_limits::max() + : (1ULL << bit_width) - 1; + std::vector values(num_values); + for (int i = 0; i < num_values; ++i) { + values[i] = static_cast((0x9E3779B9ULL * i + 0x7F4A7C15ULL) & mask); + } + return values; +} + +template +void pack_values(const std::vector& values, int bit_width, faststring* packed) { + BitWriter writer(packed); + for (T value : values) { + writer.PutValue(value, bit_width); + } + writer.Flush(); +} + +template +void test_all_bit_widths(int num_values) { + for (int bit_width = 1; bit_width <= std::numeric_limits::digits; ++bit_width) { + std::vector expected = make_values(bit_width, num_values); + faststring packed; + pack_values(expected, bit_width, &packed); + std::vector actual(num_values); + + auto [end, values_read] = + BitPacking::UnpackValues(bit_width, reinterpret_cast(packed.data()), + packed.size(), num_values, actual.data()); + + EXPECT_EQ(values_read, num_values) << "bit_width=" << bit_width; + EXPECT_EQ(end, reinterpret_cast(packed.data()) + packed.size()) + << "bit_width=" << bit_width; + EXPECT_EQ(actual, expected) << "bit_width=" << bit_width; + } +} + +template +void test_truncated_input() { + constexpr int num_values = 160; + constexpr T sentinel = std::numeric_limits::max(); + for (int bit_width = 1; bit_width <= std::numeric_limits::digits; ++bit_width) { + std::vector expected = make_values(bit_width, num_values); + faststring packed; + pack_values(expected, bit_width, &packed); + const auto* packed_data = reinterpret_cast(packed.data()); + std::vector input(packed_data, packed_data + packed.size() - 1); + const int64_t input_bytes = input.size(); + const int64_t expected_values_read = input_bytes * 8 / bit_width; + const int64_t expected_bytes_read = (expected_values_read * bit_width + 7) / 8; + std::vector actual(num_values, sentinel); + + auto [end, values_read] = BitPacking::UnpackValues(bit_width, input.data(), input_bytes, + num_values, actual.data()); + + EXPECT_EQ(values_read, expected_values_read) << "bit_width=" << bit_width; + EXPECT_EQ(end, input.data() + expected_bytes_read) << "bit_width=" << bit_width; + EXPECT_TRUE(std::equal(expected.begin(), expected.begin() + expected_values_read, + actual.begin())) + << "bit_width=" << bit_width; + EXPECT_TRUE(std::all_of(actual.begin() + expected_values_read, actual.end(), + [](T value) { return value == sentinel; })) + << "bit_width=" << bit_width; + } +} + +TEST(BitPackingTest, PdepUnpackAllBitWidths) { + for (int num_values : {127, 128, 129}) { + test_all_bit_widths(num_values); + test_all_bit_widths(num_values); + test_all_bit_widths(num_values); + } +} + +TEST(BitPackingTest, PdepUnpackTruncatedInput) { + test_truncated_input(); + test_truncated_input(); + test_truncated_input(); +} + +#if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +TEST(BitPackingTest, DisableBmi2Optimizations) { + const bool was_enabled = config::enable_bmi2_optimizations; + config::enable_bmi2_optimizations = false; + EXPECT_FALSE(PdepUnpack::is_supported()); + config::enable_bmi2_optimizations = was_enabled; +} +#endif + +} // namespace +} // namespace doris diff --git a/be/test/util/blocking_queue_test.cpp b/be/test/util/blocking_queue_test.cpp index 419ffe49ca1f11..00cf891fdbaf65 100644 --- a/be/test/util/blocking_queue_test.cpp +++ b/be/test/util/blocking_queue_test.cpp @@ -21,8 +21,15 @@ #include #include +#include +#include +#include #include #include +#include + +#include "cpp/sync_point.h" +#include "util/defer_op.h" namespace doris { @@ -142,4 +149,313 @@ TEST(BlockingQueueTest, TestMultipleThreads) { test.Run(); } +// Coordinates the queue threads through callbacks installed at the sync points below. The +// waiter callback runs while the queue mutex is still held, immediately before wait_for(). The +// operation callback runs after the peer operation has acquired that same mutex. Holding the +// first peer operation there past the wait timeout deterministically creates this ordering: +// +// waiter times out -> peer changes the queue and unlocks -> waiter reacquires the mutex +// +// This is the ordering that made the old notifier and the timed waiter decrement the same waiter +// registration. Later peer operations are not blocked, so the test can verify whether a real +// waiter receives the notification that follows the corrupted registration. +class GetPutRaceCoordinator { +public: + // The first callback is the waiter used to create the counter corruption. The second callback + // is the waiter used to verify that the next notification is not lost. + void get_waiter_registered() { + std::lock_guard lock(_get_lock); + ++_get_waiter_count; + _get_cv.notify_all(); + } + + void put_waiter_registered() { + std::lock_guard lock(_put_lock); + ++_put_waiter_count; + _put_cv.notify_all(); + } + + void block_first_get() { + std::unique_lock lock(_get_lock); + ++_get_operation_count; + // Only the first get must hold the queue mutex across the put timeout. The later get must + // proceed normally to exercise BlockingQueue's put notification decision. + if (_get_operation_count != 1) { + return; + } + _first_get_has_lock = true; + _get_cv.notify_all(); + _get_cv.wait(lock, [&] { return _release_get; }); + } + + void block_first_put() { + std::unique_lock lock(_put_lock); + ++_put_operation_count; + // Only the first put must hold the queue mutex across the get timeout. The later put must + // proceed normally to exercise BlockingQueue's get notification decision. + if (_put_operation_count != 1) { + return; + } + _first_put_has_lock = true; + _put_cv.notify_all(); + _put_cv.wait(lock, [&] { return _release_put; }); + } + + template + bool wait_for_get_waiter_count(int expected, + const std::chrono::duration& timeout) { + std::unique_lock lock(_get_lock); + return _get_cv.wait_for(lock, timeout, [&] { return _get_waiter_count >= expected; }); + } + + template + bool wait_for_put_waiter_count(int expected, + const std::chrono::duration& timeout) { + std::unique_lock lock(_put_lock); + return _put_cv.wait_for(lock, timeout, [&] { return _put_waiter_count >= expected; }); + } + + template + bool wait_for_first_get_lock(const std::chrono::duration& timeout) { + std::unique_lock lock(_get_lock); + return _get_cv.wait_for(lock, timeout, [&] { return _first_get_has_lock; }); + } + + template + bool wait_for_first_put_lock(const std::chrono::duration& timeout) { + std::unique_lock lock(_put_lock); + return _put_cv.wait_for(lock, timeout, [&] { return _first_put_has_lock; }); + } + + void release_blocked_get() { + std::lock_guard lock(_get_lock); + _release_get = true; + _get_cv.notify_all(); + } + + void release_blocked_put() { + std::lock_guard lock(_put_lock); + _release_put = true; + _put_cv.notify_all(); + } + +private: + // Coordinates get waiter registration and a get operation that holds the queue mutex. This + // state is independent from the put path so each callback changes only get-related state. + std::mutex _get_lock; + std::condition_variable _get_cv; + int _get_waiter_count = 0; + int _get_operation_count = 0; + bool _first_get_has_lock = false; + bool _release_get = false; + + // Coordinates put waiter registration and a put operation that holds the queue mutex. + std::mutex _put_lock; + std::condition_variable _put_cv; + int _put_waiter_count = 0; + int _put_operation_count = 0; + bool _first_put_has_lock = false; + bool _release_put = false; +}; + +class BlockingQueueWaiterTest : public testing::Test { +protected: + void SetUp() override { + _sync_point = SyncPoint::get_instance(); + _sync_point->clear_all_call_backs(); + _sync_point->clear_trace(); + _sync_point->enable_processing(); + } + + void TearDown() override { + _sync_point->disable_processing(); + _sync_point->clear_all_call_backs(); + _sync_point->clear_trace(); + } + + SyncPoint* _sync_point = nullptr; +}; + +// Verifies that a timed get remains the sole owner of its _get_waiting registration. The first +// phase forces try_put to hold the queue mutex after the get timeout expires; the old code made +// try_put and the timed get both decrement the same registration, wrapping the counter to +// SIZE_MAX. The second phase registers another get and verifies that a following try_put observes +// that waiter and wakes it immediately instead of leaving it blocked until the timeout fallback. +TEST_F(BlockingQueueWaiterTest, TimedGetNotificationRace) { + // The first wait is deliberately short so the producer can hold the queue mutex past its + // deadline. The second wait has a 30-second fallback, while the assertion waits only 5 + // seconds; therefore, a ready future proves that notify_one() woke it rather than its timeout. + constexpr int64_t kTimedGetWaitTimeoutMs = 100; + constexpr int64_t kNotifiedGetWaitTimeoutMs = 30 * 1000; + constexpr auto kCoordinationTimeout = std::chrono::seconds(5); + constexpr auto kGetNotificationDeadline = std::chrono::seconds(5); + + BlockingQueue get_queue(1); + GetPutRaceCoordinator get_race; + _sync_point->set_call_back("BlockingQueue::controlled_blocking_get::before_wait", + [&](auto&&) { get_race.get_waiter_registered(); }); + _sync_point->set_call_back("BlockingQueue::try_put::after_lock", + [&](auto&&) { get_race.block_first_put(); }); + + std::promise timed_get_promise; + auto timed_get_future = timed_get_promise.get_future(); + std::promise try_put_promise; + auto try_put_future = try_put_promise.get_future(); + std::promise notified_get_promise; + auto notified_get_future = notified_get_promise.get_future(); + std::vector get_threads; + // Keep this guard after every promise and future declaration. On a fatal assertion it runs + // first, releases either blocked callback, shuts down the queue, and joins all workers before + // their referenced asynchronous state is destroyed. + Defer cleanup {[&] { + get_race.release_blocked_put(); + get_queue.shutdown(); + for (auto& thread : get_threads) { + thread.join(); + } + }}; + + get_threads.emplace_back([&] { + int32_t get_value = -1; + get_queue.controlled_blocking_get(&get_value, kTimedGetWaitTimeoutMs); + timed_get_promise.set_value(get_value); + }); + ASSERT_TRUE(get_race.wait_for_get_waiter_count(1, kCoordinationTimeout)); + EXPECT_EQ(get_queue.get_waiting_count_for_test(), 1); + EXPECT_EQ(get_queue.put_waiting_count_for_test(), 0); + + // Phase 1: timed_get has registered _get_waiting=1 and released the queue mutex in + // wait_for(). try_put then acquires that mutex and is stopped by the callback. Keeping try_put + // there for twice the get timeout guarantees that timed_get expires while it is unable to + // reacquire the mutex. + get_threads.emplace_back([&] { try_put_promise.set_value(get_queue.try_put(1)); }); + ASSERT_TRUE(get_race.wait_for_first_put_lock(kCoordinationTimeout)); + + std::this_thread::sleep_for(std::chrono::milliseconds(2 * kTimedGetWaitTimeoutMs)); + get_race.release_blocked_put(); + // In the old implementation try_put decremented _get_waiting from 1 to 0 before notifying, + // then timed_get reacquired the mutex and decremented the same registration again because + // wait_for() returned timeout. The size_t counter consequently wrapped from 0 to SIZE_MAX. In + // the fixed implementation only timed_get owns the registration, so the counter returns to + // zero. + ASSERT_EQ(try_put_future.wait_for(kCoordinationTimeout), std::future_status::ready); + EXPECT_TRUE(try_put_future.get()); + ASSERT_EQ(timed_get_future.wait_for(kCoordinationTimeout), std::future_status::ready); + EXPECT_EQ(timed_get_future.get(), 1); + EXPECT_EQ(get_queue.get_waiting_count_for_test(), 0); + EXPECT_EQ(get_queue.put_waiting_count_for_test(), 0); + + // Phase 2: in the old implementation notified_get changed _get_waiting from SIZE_MAX to 0. The + // following try_put therefore observed no get waiter and skipped notify_one(), leaving + // notified_get asleep until its 30-second fallback. With correct accounting notified_get + // changes the counter from 0 to 1 and try_put wakes it within the 5-second notification + // deadline. + get_threads.emplace_back([&] { + int32_t get_value = -1; + get_queue.controlled_blocking_get(&get_value, kNotifiedGetWaitTimeoutMs); + notified_get_promise.set_value(get_value); + }); + ASSERT_TRUE(get_race.wait_for_get_waiter_count(2, kCoordinationTimeout)); + EXPECT_EQ(get_queue.get_waiting_count_for_test(), 1); + EXPECT_EQ(get_queue.put_waiting_count_for_test(), 0); + + EXPECT_TRUE(get_queue.try_put(2)); + ASSERT_EQ(notified_get_future.wait_for(kGetNotificationDeadline), std::future_status::ready); + EXPECT_EQ(notified_get_future.get(), 2); + EXPECT_EQ(get_queue.get_waiting_count_for_test(), 0); + EXPECT_EQ(get_queue.put_waiting_count_for_test(), 0); +} + +// Verifies the symmetric rule for a timed put and its _put_waiting registration. The first phase +// forces blocking_get to hold the queue mutex after the put timeout expires; the old code made the +// get and timed put both decrement the registration and wrap the counter. The second phase +// registers another put, removes an item with blocking_get, and verifies that the put is notified +// immediately and can add its item without waiting for the timeout fallback. +TEST_F(BlockingQueueWaiterTest, TimedPutNotificationRace) { + // As in TimedGetNotificationRace, the 30-second second wait and 5-second assertion deadline + // distinguish a real notification from the periodic timeout fallback. + constexpr int64_t kTimedPutWaitTimeoutMs = 100; + constexpr int64_t kNotifiedPutWaitTimeoutMs = 30 * 1000; + constexpr auto kCoordinationTimeout = std::chrono::seconds(5); + constexpr auto kPutNotificationDeadline = std::chrono::seconds(5); + + BlockingQueue put_queue(1); + ASSERT_TRUE(put_queue.try_put(0)); + + GetPutRaceCoordinator put_race; + _sync_point->set_call_back("BlockingQueue::controlled_blocking_put::before_wait", + [&](auto&&) { put_race.put_waiter_registered(); }); + _sync_point->set_call_back("BlockingQueue::controlled_blocking_get::after_lock", + [&](auto&&) { put_race.block_first_get(); }); + + std::promise timed_put_promise; + auto timed_put_future = timed_put_promise.get_future(); + std::promise blocking_get_promise; + auto blocking_get_future = blocking_get_promise.get_future(); + std::promise notified_put_promise; + auto notified_put_future = notified_put_promise.get_future(); + std::vector put_threads; + // The guard is declared last so every failure path joins the workers before their promises and + // futures are destroyed. + Defer cleanup {[&] { + put_race.release_blocked_get(); + put_queue.shutdown(); + for (auto& thread : put_threads) { + thread.join(); + } + }}; + + put_threads.emplace_back([&] { + timed_put_promise.set_value(put_queue.controlled_blocking_put(1, kTimedPutWaitTimeoutMs)); + }); + ASSERT_TRUE(put_race.wait_for_put_waiter_count(1, kCoordinationTimeout)); + EXPECT_EQ(put_queue.get_waiting_count_for_test(), 0); + EXPECT_EQ(put_queue.put_waiting_count_for_test(), 1); + + // Phase 1 is the put-side mirror of the get race. timed_put registers _put_waiting=1 and waits + // because the queue contains 0. blocking_get then acquires and holds the queue mutex until the + // put timeout has expired, preventing timed_put from returning from wait_for(). + put_threads.emplace_back([&] { + int32_t get_value = -1; + put_queue.blocking_get(&get_value); + blocking_get_promise.set_value(get_value); + }); + ASSERT_TRUE(put_race.wait_for_first_get_lock(kCoordinationTimeout)); + + std::this_thread::sleep_for(std::chrono::milliseconds(2 * kTimedPutWaitTimeoutMs)); + put_race.release_blocked_get(); + // Previously blocking_get decremented _put_waiting when it removed 0, and timed_put decremented + // it again after reacquiring the mutex with a timeout result. The counter wrapped to SIZE_MAX + // before timed_put put 1. With single-owner accounting timed_put performs the only decrement + // and leaves the counter at zero. + ASSERT_EQ(blocking_get_future.wait_for(kCoordinationTimeout), std::future_status::ready); + EXPECT_EQ(blocking_get_future.get(), 0); + ASSERT_EQ(timed_put_future.wait_for(kCoordinationTimeout), std::future_status::ready); + EXPECT_TRUE(timed_put_future.get()); + EXPECT_EQ(put_queue.get_waiting_count_for_test(), 0); + EXPECT_EQ(put_queue.put_waiting_count_for_test(), 0); + + // Phase 2: the old SIZE_MAX counter wrapped to zero when notified_put registered. The following + // blocking_get therefore skipped notify_one(), so notified_put could not put 2 before the + // 5-second deadline. The fixed counter is one, causing blocking_get to wake notified_put + // immediately and allowing the final value check to finish. + put_threads.emplace_back([&] { + notified_put_promise.set_value( + put_queue.controlled_blocking_put(2, kNotifiedPutWaitTimeoutMs)); + }); + ASSERT_TRUE(put_race.wait_for_put_waiter_count(2, kCoordinationTimeout)); + EXPECT_EQ(put_queue.get_waiting_count_for_test(), 0); + EXPECT_EQ(put_queue.put_waiting_count_for_test(), 1); + + int32_t get_value = -1; + EXPECT_TRUE(put_queue.blocking_get(&get_value)); + EXPECT_EQ(get_value, 1); + ASSERT_EQ(notified_put_future.wait_for(kPutNotificationDeadline), std::future_status::ready); + EXPECT_TRUE(notified_put_future.get()); + EXPECT_EQ(put_queue.get_waiting_count_for_test(), 0); + EXPECT_EQ(put_queue.put_waiting_count_for_test(), 0); + EXPECT_TRUE(put_queue.blocking_get(&get_value)); + EXPECT_EQ(get_value, 2); +} + } // namespace doris diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 941e8c5a52daa4..2de1145ac6f00d 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -31,6 +31,7 @@ OPTS="$(getopt \ -l 'daemon' \ -l 'helper:' \ -l 'image:' \ + -l 'local_resource_group:' \ -l 'version' \ -l 'metadata_failure_recovery' \ -l 'recovery_journal_id:' \ @@ -47,6 +48,7 @@ HELPER='' IMAGE_PATH='' IMAGE_TOOL='' OPT_VERSION='' +declare -a LOCAL_RESOURCE_GROUP_ARGS=() declare -a HELPER_ARGS=() declare -a METADATA_FAILURE_RECOVERY_ARGS=() declare -a RECOVERY_JOURNAL_ID_ARGS=() @@ -83,6 +85,10 @@ while true; do IMAGE_PATH="$2" shift 2 ;; + --local_resource_group) + LOCAL_RESOURCE_GROUP_ARGS=("--local_resource_group" "$2") + shift 2 + ;; --cluster_snapshot) CLUSTER_SNAPSHOT_ARGS=("--cluster_snapshot" "$2") shift 2 @@ -428,23 +434,23 @@ fi if [[ "${OPT_VERSION}" != "" ]]; then export DORIS_LOG_TO_STDERR=1 - ${LIMIT:+${LIMIT}} "${JAVA}" org.apache.doris.DorisFE --version + ${LIMIT:+${LIMIT}} "${JAVA}" org.apache.doris.DorisFE "${LOCAL_RESOURCE_GROUP_ARGS[@]}" --version exit 0 fi if [[ "${IMAGE_TOOL}" -eq 1 ]]; then if [[ -n "${IMAGE_PATH}" ]]; then - ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE -i "${IMAGE_PATH}" + ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE "${LOCAL_RESOURCE_GROUP_ARGS[@]}" -i "${IMAGE_PATH}" else echo "Internal error, USE IMAGE_TOOL like: ./start_fe.sh --image image_path" fi elif [[ "${RUN_DAEMON}" -eq 1 ]]; then - nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE "${HELPER_ARGS[@]}" "${METADATA_FAILURE_RECOVERY_ARGS[@]}" "${RECOVERY_JOURNAL_ID_ARGS[@]}" "${CLUSTER_SNAPSHOT_ARGS[@]}" "${DROP_BACKENDS_ARGS[@]}" "$@" >>"${STDOUT_LOGGER}" 2>&1 >"${STDOUT_LOGGER}" 2>&1 >"${STDOUT_LOGGER}" >"${STDOUT_LOGGER}" >"${STDOUT_LOGGER}" 2>&1 >"${STDOUT_LOGGER}" 2>&1 $0 --be --fe build Backend, Frontend, and Java UDF library $0 --be --coverage build Backend with coverage enabled $0 --be --output PATH build Backend, the result will be output to PATH(relative paths are available) - $0 --be-extension-ignore avro-scanner build be-java-extensions, choose which modules to ignore. Multiple modules separated by commas, like --be-extension-ignore avro-scanner,hadoop-hudi-scanner + $0 --be-extension-ignore paimon-scanner build be-java-extensions, choose which modules to ignore. Multiple modules separated by commas, like --be-extension-ignore paimon-scanner,hadoop-hudi-scanner USE_AVX2=0 $0 --be build Backend and not using AVX2 instruction. USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2 instruction, and strip the debug info for Backend @@ -717,7 +717,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Filesystem API and SPI plugin modules (loaded at runtime as plugins) modules+=("fe-filesystem/fe-filesystem-api") modules+=("fe-filesystem/fe-filesystem-spi") - for _fs_mod in s3 oss cos obs azure hdfs local broker http; do + for _fs_mod in s3 oss cos obs azure hdfs-base hdfs oss-hdfs jfs local broker http; do if [[ -d "${DORIS_HOME}/fe/fe-filesystem/fe-filesystem-${_fs_mod}" ]]; then modules+=("fe-filesystem/fe-filesystem-${_fs_mod}") fi @@ -748,7 +748,6 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then modules+=("be-java-extensions/paimon-scanner") modules+=("be-java-extensions/trino-connector-scanner") modules+=("be-java-extensions/max-compute-connector") - modules+=("be-java-extensions/avro-scanner") # lakesoul-scanner has been deprecated # modules+=("be-java-extensions/lakesoul-scanner") modules+=("be-java-extensions/preload-extensions") @@ -1049,7 +1048,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # Deploy filesystem provider plugins as independent plugin directories # Each sub-directory is one storage backend loaded at runtime by FileSystemPluginManager. FS_PLUGIN_DIR="${DORIS_OUTPUT}/fe/plugins/filesystem" - for fs_module in s3 azure oss cos obs hdfs local broker http; do + for fs_module in s3 azure oss cos obs hdfs oss-hdfs jfs local broker http; do fs_plugin_target="${FS_PLUGIN_DIR}/${fs_module}" fs_module_dir="${DORIS_HOME}/fe/fe-filesystem/fe-filesystem-${fs_module}" if [ ! -d "${fs_module_dir}" ]; then @@ -1169,7 +1168,6 @@ EOF extensions_modules+=("paimon-scanner") extensions_modules+=("trino-connector-scanner") extensions_modules+=("max-compute-connector") - extensions_modules+=("avro-scanner") # lakesoul-scanner has been deprecated # extensions_modules+=("lakesoul-scanner") extensions_modules+=("preload-extensions") diff --git a/cloud/script/run_all_tests.sh b/cloud/script/run_all_tests.sh index 71290198a7f365..466c2cb08136cd 100644 --- a/cloud/script/run_all_tests.sh +++ b/cloud/script/run_all_tests.sh @@ -134,6 +134,13 @@ function report_coverage() { } export LSAN_OPTIONS=suppressions=./lsan_suppr.conf + +function is_fdb_asan_failure() { + local test_log=$1 + grep -Fq "libfdb_c.so" "${test_log}" && + grep -Eq "AddressSanitizer:DEADLYSIGNAL|ERROR: AddressSanitizer" "${test_log}" +} + unittest_files=() ret=0 if [[ "${filter}" != "" ]]; then @@ -157,9 +164,15 @@ for i in *_test; do continue fi - LLVM_PROFILE_FILE="./report/${i}.profraw" "./${i}" --gtest_print_time=true --gtest_output="xml:${i}.xml" "${filter}" - last_ret=$? + test_log=$(mktemp) + LLVM_PROFILE_FILE="./report/${i}.profraw" "./${i}" --gtest_print_time=true --gtest_output="xml:${i}.xml" "${filter}" 2>&1 | tee "${test_log}" + last_ret=${PIPESTATUS[0]} echo "${i} ret=${last_ret}" + if [[ ${last_ret} -ne 0 ]] && is_fdb_asan_failure "${test_log}"; then + echo "========== ${i} failed in libfdb_c.so under ASAN, treating as pass ==========" + last_ret=0 + fi + rm -f "${test_log}" if [[ ${ret} -eq 0 ]]; then ret=${last_ret} fi diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 29e795ca078423..a4a2c17441ba75 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -282,6 +282,9 @@ CONF_mBool(enable_s3_rate_limiter, "false"); // s3_rate_limit_inject_probility is the probability (0-100) of injecting a rate limit error. CONF_mBool(enable_s3_rate_limit_inject, "false"); CONF_mInt32(s3_rate_limit_inject_probility, "30"); +// Log active S3 rate limiter every N throttled/rejected requests, 0 means no log. +CONF_mInt64(s3_rate_limiter_log_interval, "1000"); +CONF_Validator(s3_rate_limiter_log_interval, [](int64_t config) -> bool { return config >= 0; }); CONF_mInt64(s3_get_bucket_tokens, "1000000000000000000"); CONF_Validator(s3_get_bucket_tokens, [](int64_t config) -> bool { return config > 0; }); @@ -332,8 +335,6 @@ CONF_Validator(s3_client_http_scheme, [](const std::string& config) -> bool { // Max retry times for object storage request CONF_mInt64(max_s3_client_retry, "10"); -// Whether to retry on S3 SlowDown (429/503) errors -CONF_Bool(s3_client_retry_slow_down, "false"); // Max byte getting delete bitmap can return, default is 1GB CONF_mInt64(max_get_delete_bitmap_byte, "1073741824"); diff --git a/cloud/src/recycler/azure_obj_client.cpp b/cloud/src/recycler/azure_obj_client.cpp index 94901cde48b6c1..5390c1c5f99608 100644 --- a/cloud/src/recycler/azure_obj_client.cpp +++ b/cloud/src/recycler/azure_obj_client.cpp @@ -35,6 +35,7 @@ #include "common/config.h" #include "common/logging.h" #include "common/stopwatch.h" +#include "cpp/obj_retry_strategy.h" #include "cpp/sync_point.h" #include "cpp/token_bucket_rate_limiter.h" #include "recycler/s3_accessor.h" @@ -56,7 +57,9 @@ auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) { if (!config::enable_s3_rate_limiter) { return callback(); } - auto sleep_duration = AccessorRateLimiter::instance().rate_limiter(op)->add(1); + auto sleep_duration = + doris::apply_s3_rate_limit(op, AccessorRateLimiter::instance().rate_limiter(op), + config::s3_rate_limiter_log_interval); if (sleep_duration < 0) { throw std::runtime_error("Azure exceeds request limit"); } @@ -81,6 +84,7 @@ ObjectStorageResponse do_azure_client_call(Func f, std::string_view url, std::st try { f(); } catch (Azure::Core::RequestFailedException& e) { + doris::record_object_request_failed(static_cast(e.StatusCode)); auto msg = fmt::format( "Azure request failed because {}, http_code: {}, request_id: {}, url: {}, " "key: {}", @@ -280,6 +284,7 @@ ObjectStorageResponse AzureObjClient::delete_objects(const std::string& bucket, 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) { continue; } + doris::record_object_request_failed(static_cast(e.StatusCode)); auto msg = fmt::format( "Azure request failed because {}, http code {}, request id {}, url {}", e.Message, static_cast(e.StatusCode), e.RequestId, client_->GetUrl()); @@ -333,4 +338,4 @@ ObjectStorageResponse AzureObjClient::abort_multipart_upload(ObjectStoragePathRe return delete_object(path); } -} // namespace doris::cloud \ No newline at end of file +} // namespace doris::cloud diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index bd1b9bff878cee..c2b0e6b7c176eb 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -4358,16 +4358,15 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext& TEST_SYNC_POINT_CALLBACK("InstanceRecycler::recycle_tablet.create_rowset_meta", &resp); for (const auto& rs_meta : resp.rowset_meta()) { - // Empty rowsets have no segment objects to delete, so they do not need a resource id. - if (rs_meta.num_segments() <= 0) { - LOG_INFO("rowset meta has no segments, skip this rowset") - .tag("rs_meta", rs_meta.ShortDebugString()) - .tag("instance_id", instance_id_) - .tag("tablet_id", tablet_id); - recycle_rowsets_number += 1; - continue; - } if (!rs_meta.has_resource_id() || rs_meta.resource_id().empty()) { + if (rs_meta.num_segments() <= 0) { + LOG_INFO("rowset meta has no segments and no resource id, skip this rowset") + .tag("rs_meta", rs_meta.ShortDebugString()) + .tag("instance_id", instance_id_) + .tag("tablet_id", tablet_id); + recycle_rowsets_number += 1; + continue; + } LOG_WARNING("rowset meta has a missing or empty resource id, impossible!") .tag("rs_meta", rs_meta.ShortDebugString()) .tag("instance_id", instance_id_) diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp index 51cc9023904aa5..7c4fde9704110e 100644 --- a/cloud/src/recycler/s3_accessor.cpp +++ b/cloud/src/recycler/s3_accessor.cpp @@ -73,22 +73,15 @@ bvar::LatencyRecorder s3_get_bucket_version_latency("s3_get_bucket_version"); bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); }; // namespace s3_bvar -bvar::Adder get_rate_limit_ns("get_rate_limit_ns"); -bvar::Adder get_rate_limit_exceed_req_num("get_rate_limit_exceed_req_num"); -bvar::Adder put_rate_limit_ns("put_rate_limit_ns"); -bvar::Adder put_rate_limit_exceed_req_num("put_rate_limit_exceed_req_num"); - AccessorRateLimiter::AccessorRateLimiter() - : _rate_limiters( - {std::make_unique( - config::s3_get_token_per_second, config::s3_get_bucket_tokens, - config::s3_get_token_limit, - metric_func_factory(get_rate_limit_ns, get_rate_limit_exceed_req_num)), - std::make_unique( - config::s3_put_token_per_second, config::s3_put_bucket_tokens, - config::s3_put_token_limit, - metric_func_factory(put_rate_limit_ns, - put_rate_limit_exceed_req_num))}) {} + : _rate_limiters({std::make_unique( + config::s3_get_token_per_second, config::s3_get_bucket_tokens, + config::s3_get_token_limit, + s3_rate_limiter_metric_func(S3RateLimitType::GET)), + std::make_unique( + config::s3_put_token_per_second, config::s3_put_bucket_tokens, + config::s3_put_token_limit, + s3_rate_limiter_metric_func(S3RateLimitType::PUT))}) {} S3RateLimiterHolder* AccessorRateLimiter::rate_limiter(S3RateLimitType type) { CHECK(type == S3RateLimitType::GET || type == S3RateLimitType::PUT) << to_string(type); @@ -394,9 +387,6 @@ int S3Accessor::init() { case S3Conf::AZURE: { #ifdef USE_AZURE Azure::Storage::Blobs::BlobClientOptions options; - if (config::s3_client_retry_slow_down) { - options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests); - } options.Retry.MaxRetries = config::max_s3_client_retry; auto cred = std::make_shared(conf_.ak, conf_.sk); @@ -441,8 +431,9 @@ int S3Accessor::init() { if (config::s3_client_http_scheme == "http") { aws_config.scheme = Aws::Http::Scheme::HTTP; } + // Recycler should fail fast on S3 SlowDown instead of retrying and blocking worker threads. aws_config.retryStrategy = std::make_shared( - config::max_s3_client_retry, config::s3_client_retry_slow_down); + config::max_s3_client_retry, /*retry_slow_down=*/false); if (_ca_cert_file_path.empty()) { _ca_cert_file_path = @@ -451,6 +442,11 @@ int S3Accessor::init() { if (!_ca_cert_file_path.empty()) { aws_config.caFile = _ca_cert_file_path; } + // Mirror BE PR #49315: default ClientConfiguration leaves requestTimeoutMs=3000, + // which the vendored aws-sdk-cpp maps to CURLOPT_LOW_SPEED_TIME=3 and causes + // curl error 28 on slow/large S3 DeleteObjects (OVH cold vault). + aws_config.requestTimeoutMs = 30000; + aws_config.connectTimeoutMs = 5000; auto s3_client = std::make_shared( get_aws_credentials_provider(conf_), std::move(aws_config), Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, diff --git a/cloud/src/recycler/s3_obj_client.cpp b/cloud/src/recycler/s3_obj_client.cpp index c45a6aae168c0c..3f299d0fef03e5 100644 --- a/cloud/src/recycler/s3_obj_client.cpp +++ b/cloud/src/recycler/s3_obj_client.cpp @@ -32,6 +32,7 @@ #include "common/config.h" #include "common/logging.h" #include "common/stopwatch.h" +#include "cpp/obj_retry_strategy.h" #include "cpp/sync_point.h" #include "cpp/token_bucket_rate_limiter.h" #include "recycler/s3_accessor.h" @@ -43,6 +44,10 @@ namespace doris::cloud { return {Aws::S3::S3Errors::INTERNAL_FAILURE, "exceeds limit", "exceeds limit", false}; } +void record_s3_request_failed(const Aws::S3::S3Error& error) { + doris::record_object_request_failed(static_cast(error.GetResponseCode())); +} + template auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) { using T = decltype(callback()); @@ -55,7 +60,9 @@ auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) { if (!config::enable_s3_rate_limiter) { return callback(); } - auto sleep_duration = AccessorRateLimiter::instance().rate_limiter(op)->add(1); + auto sleep_duration = + doris::apply_s3_rate_limit(op, AccessorRateLimiter::instance().rate_limiter(op), + config::s3_rate_limiter_log_interval); if (sleep_duration < 0) { return T(s3_error_factory()); } @@ -118,6 +125,7 @@ class S3ObjListIterator final : public ObjectListIterator { return false; } + record_s3_request_failed(outcome.GetError()); LOG_WARNING("failed to list objects") .tag("endpoint", endpoint_) .tag("bucket", req_.GetBucket()) @@ -210,6 +218,7 @@ ObjectStorageResponse S3ObjClient::put_object(ObjectStoragePathRef path, std::st return s3_client_->PutObject(request); }); if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); LOG_WARNING("failed to put object") .tag("endpoint", endpoint_) .tag("bucket", path.bucket) @@ -237,6 +246,7 @@ ObjectStorageResponse S3ObjClient::head_object(ObjectStoragePathRef path, Object } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { return 1; } else { + record_s3_request_failed(outcome.GetError()); LOG_WARNING("failed to head object") .tag("endpoint", endpoint_) .tag("bucket", path.bucket) @@ -276,6 +286,7 @@ ObjectStorageResponse S3ObjClient::delete_objects(const std::string& bucket, return s3_client_->DeleteObjects(delete_request); }); if (!delete_outcome.IsSuccess()) { + record_s3_request_failed(delete_outcome.GetError()); LOG_WARNING("failed to delete objects") .tag("endpoint", endpoint_) .tag("bucket", bucket) @@ -326,6 +337,10 @@ ObjectStorageResponse S3ObjClient::delete_object(ObjectStoragePathRef path) { }); TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_object", &outcome); if (!outcome.IsSuccess()) { + if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + return {ObjectStorageResponse::NOT_FOUND, outcome.GetError().GetMessage()}; + } + record_s3_request_failed(outcome.GetError()); LOG_WARNING("failed to delete object") .tag("endpoint", endpoint_) .tag("bucket", path.bucket) @@ -334,9 +349,6 @@ ObjectStorageResponse S3ObjClient::delete_object(ObjectStoragePathRef path) { .tag("error", outcome.GetError().GetMessage()) .tag("exception", outcome.GetError().GetExceptionName()) .tag("request_id", outcome.GetError().GetRequestId()); - if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return {ObjectStorageResponse::NOT_FOUND, outcome.GetError().GetMessage()}; - } return {ObjectStorageResponse::UNDEFINED, outcome.GetError().GetMessage()}; } return {ObjectStorageResponse::OK}; @@ -365,6 +377,7 @@ ObjectStorageResponse S3ObjClient::get_life_cycle(const std::string& bucket, } } } else { + record_s3_request_failed(outcome.GetError()); LOG_WARNING("Err for check interval: failed to get bucket lifecycle") .tag("endpoint", endpoint_) .tag("bucket", bucket) @@ -397,6 +410,7 @@ ObjectStorageResponse S3ObjClient::check_versioning(const std::string& bucket) { return -1; } } else { + record_s3_request_failed(outcome.GetError()); LOG_WARNING("Err for check interval: failed to get status of bucket versioning") .tag("endpoint", endpoint_) .tag("bucket", bucket) @@ -414,6 +428,10 @@ ObjectStorageResponse S3ObjClient::abort_multipart_upload(ObjectStoragePathRef p request.WithBucket(path.bucket).WithKey(path.key).WithUploadId(upload_id); auto outcome = s3_put_rate_limit([&]() { return s3_client_->AbortMultipartUpload(request); }); if (!outcome.IsSuccess()) { + if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + return {ObjectStorageResponse::OK}; + } + record_s3_request_failed(outcome.GetError()); LOG_WARNING("failed to abort multipart upload") .tag("endpoint", endpoint_) .tag("bucket", path.bucket) @@ -423,9 +441,6 @@ ObjectStorageResponse S3ObjClient::abort_multipart_upload(ObjectStoragePathRef p .tag("error", outcome.GetError().GetMessage()) .tag("exception", outcome.GetError().GetExceptionName()) .tag("request_id", outcome.GetError().GetRequestId()); - if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return {ObjectStorageResponse::OK}; - } return {ObjectStorageResponse::UNDEFINED, outcome.GetError().GetMessage()}; } return {ObjectStorageResponse::OK}; diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index ad8075cec9c55f..78d5b388bd8700 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -7421,6 +7421,44 @@ TEST(RecyclerTest, recycle_tablet_with_empty_resource_id_and_no_segments) { EXPECT_EQ(recycler.recycle_tablet(0, ctx), 0); } +TEST(RecyclerTest, recycle_tablet_with_resource_id_and_no_segments) { + auto* sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); + }; + + auto txn_kv = std::make_shared(); + EXPECT_EQ(txn_kv->init(), 0); + InstanceInfoPB instance; + instance.set_instance_id("test_instance"); + + auto accessor = std::make_shared(); + constexpr int64_t tablet_id = 1234; + EXPECT_EQ(accessor->put_file("data/1234/orphan.dat", ""), 0); + + sp->set_call_back("InstanceRecycler::recycle_tablet.create_rowset_meta", [](auto&& args) { + auto* resp = try_any_cast(args[0]); + auto* rs = resp->add_rowset_meta(); + rs->set_num_segments(0); + rs->set_resource_id("resource_id"); + rs = resp->add_rowset_meta(); + rs->set_num_segments(0); + }); + sp->enable_processing(); + + InstanceRecycler recycler(txn_kv, instance, thread_group, + std::make_shared(txn_kv)); + EXPECT_EQ(recycler.init(), 0); + recycler.TEST_add_accessor("resource_id", accessor); + + EXPECT_EQ(accessor->exists("data/1234/orphan.dat"), 0); + RecyclerMetricsContext ctx; + EXPECT_EQ(recycler.recycle_tablet(tablet_id, ctx), 0); + EXPECT_EQ(accessor->exists("data/1234/orphan.dat"), 1); +} + TEST(RecyclerTest, recycle_tablet_with_wrong_resource_id) { auto* sp = SyncPoint::get_instance(); DORIS_CLOUD_DEFER { diff --git a/cloud/test/s3_rate_limiter_test.cpp b/cloud/test/s3_rate_limiter_test.cpp index b1fd5a8115fa6b..05fbb2ce35ec03 100644 --- a/cloud/test/s3_rate_limiter_test.cpp +++ b/cloud/test/s3_rate_limiter_test.cpp @@ -29,6 +29,10 @@ using namespace doris::cloud; +namespace doris { +extern bvar::Adder s3_put_rate_limit_rejected_count; +} // namespace doris + int main(int argc, char** argv) { auto conf_file = "doris_cloud.conf"; if (!doris::cloud::config::init(conf_file, true)) { @@ -107,18 +111,79 @@ TEST(S3RateLimiterTest, ExceedLimit) { } TEST(S3RateLimiterHolderTest, BvarMetric) { - bvar::Adder rate_limit_ns("rate_limit_ns"); - bvar::Adder rate_limit_exceed_req_num("rate_limit_exceed_req_num"); + bvar::Adder rate_limit_sleep_ns("rate_limit_sleep_ns"); + bvar::Adder rate_limit_sleep_count("rate_limit_sleep_count"); + bvar::Adder rate_limit_rejected_count("rate_limit_rejected_count"); auto rate_limiter_holder = doris::S3RateLimiterHolder( - 125, 250, 500, doris::metric_func_factory(rate_limit_ns, rate_limit_exceed_req_num)); + 125, 250, 251, + doris::metric_func_factory(rate_limit_sleep_ns, rate_limit_sleep_count, + &rate_limit_rejected_count)); int64_t sleep_time = rate_limiter_holder.add(250); EXPECT_EQ(sleep_time, 0); - EXPECT_EQ(rate_limit_ns.get_value(), 0); - EXPECT_EQ(rate_limit_exceed_req_num.get_value(), 0); + EXPECT_EQ(rate_limit_sleep_ns.get_value(), 0); + EXPECT_EQ(rate_limit_sleep_count.get_value(), 0); + EXPECT_EQ(rate_limit_rejected_count.get_value(), 0); sleep_time = rate_limiter_holder.add(1); - EXPECT_GT(rate_limit_ns.get_value(), 0); - EXPECT_EQ(rate_limit_exceed_req_num.get_value(), 1); EXPECT_GT(sleep_time, 0); -} \ No newline at end of file + EXPECT_GT(rate_limit_sleep_ns.get_value(), 0); + EXPECT_EQ(rate_limit_sleep_count.get_value(), 1); + EXPECT_EQ(rate_limit_rejected_count.get_value(), 0); + + sleep_time = rate_limiter_holder.add(1); + EXPECT_EQ(sleep_time, -1); + EXPECT_EQ(rate_limit_rejected_count.get_value(), 1); +} + +TEST(S3RateLimiterHolderTest, ApplyS3RateLimitRecordsRejectedMetric) { + auto rate_limiter_holder = doris::S3RateLimiterHolder( + 125, 250, 1, doris::s3_rate_limiter_metric_func(doris::S3RateLimitType::PUT)); + auto rejected_count = doris::s3_put_rate_limit_rejected_count.get_value(); + + int64_t sleep_time = + doris::apply_s3_rate_limit(doris::S3RateLimitType::PUT, &rate_limiter_holder, 1); + EXPECT_EQ(sleep_time, 0); + EXPECT_EQ(doris::s3_put_rate_limit_rejected_count.get_value(), rejected_count); + + sleep_time = doris::apply_s3_rate_limit(doris::S3RateLimitType::PUT, &rate_limiter_holder, 1); + EXPECT_EQ(sleep_time, -1); + EXPECT_EQ(doris::s3_put_rate_limit_rejected_count.get_value(), rejected_count + 1); +} + +TEST(S3RateLimiterHolderTest, ConcurrentResetReturnsConsistentConfig) { + constexpr size_t config_a_speed = 101; + constexpr size_t config_a_burst = 102; + constexpr size_t config_a_limit = 103; + constexpr size_t config_b_speed = 201; + constexpr size_t config_b_burst = 202; + constexpr size_t config_b_limit = 203; + + doris::S3RateLimiterHolder rate_limiter_holder(config_a_speed, config_a_burst, config_a_limit, + [](int64_t) {}); + std::atomic start {false}; + std::thread reset_thread([&]() { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + for (size_t i = 0; i < 10000; ++i) { + if (i % 2 == 0) { + rate_limiter_holder.reset(config_b_speed, config_b_burst, config_b_limit); + } else { + rate_limiter_holder.reset(config_a_speed, config_a_burst, config_a_limit); + } + } + }); + + start.store(true, std::memory_order_release); + for (size_t i = 0; i < 10000; ++i) { + auto result = rate_limiter_holder.add_with_config(0); + bool is_config_a = result.max_speed == config_a_speed && + result.max_burst == config_a_burst && result.limit == config_a_limit; + bool is_config_b = result.max_speed == config_b_speed && + result.max_burst == config_b_burst && result.limit == config_b_limit; + EXPECT_TRUE(is_config_a || is_config_b); + } + + reset_thread.join(); +} diff --git a/cloud/test/stopwatch_test.cpp b/cloud/test/stopwatch_test.cpp index f4ecb1d82f2b1b..bb2001f668228f 100644 --- a/cloud/test/stopwatch_test.cpp +++ b/cloud/test/stopwatch_test.cpp @@ -26,6 +26,7 @@ using namespace doris::cloud; int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } @@ -48,4 +49,4 @@ TEST(StopWatchTest, SimpleTest) { std::this_thread::sleep_for(std::chrono::microseconds(1000)); ASSERT_TRUE(s.elapsed_us() >= 1000 && s.elapsed_us() < 1500); } -} \ No newline at end of file +} diff --git a/common/cpp/obj_retry_strategy.cpp b/common/cpp/obj_retry_strategy.cpp index 58ca47f6a6808c..8399225536424c 100644 --- a/common/cpp/obj_retry_strategy.cpp +++ b/common/cpp/obj_retry_strategy.cpp @@ -24,6 +24,16 @@ namespace doris { bvar::Adder object_request_retry_count("object_request_retry_count"); +bvar::Adder s3_request_retry_too_many_requests_count( + "s3_request_retry_too_many_requests_count"); +bvar::Adder s3_request_failed_too_many_requests_count( + "s3_request_failed_too_many_requests_count"); + +void record_object_request_failed(int http_code) { + if (http_code == static_cast(Aws::Http::HttpResponseCode::TOO_MANY_REQUESTS)) { + s3_request_failed_too_many_requests_count << 1; + } +} S3CustomRetryStrategy::S3CustomRetryStrategy(int maxRetries, bool retry_slow_down) : DefaultRetryStrategy(maxRetries), _retry_slow_down(retry_slow_down) {} @@ -43,6 +53,9 @@ bool S3CustomRetryStrategy::ShouldRetry(const Aws::Client::AWSError AzureRetryRecordPolicy::Send( static_cast(response->GetStatusCode()) < 200) { if (retry_count > 0) { object_request_retry_count << 1; + if (static_cast(response->GetStatusCode()) == + static_cast(Aws::Http::HttpResponseCode::TOO_MANY_REQUESTS)) { + s3_request_retry_too_many_requests_count << 1; + } } // If the response is not successful, we log the retry attempt and status code. diff --git a/common/cpp/obj_retry_strategy.h b/common/cpp/obj_retry_strategy.h index aaf35f6b10020d..2eafe114564b21 100644 --- a/common/cpp/obj_retry_strategy.h +++ b/common/cpp/obj_retry_strategy.h @@ -25,6 +25,8 @@ #endif namespace doris { +void record_object_request_failed(int http_code); + class S3CustomRetryStrategy final : public Aws::Client::DefaultRetryStrategy { public: S3CustomRetryStrategy(int maxRetries, bool retry_slow_down = true); diff --git a/common/cpp/token_bucket_rate_limiter.cpp b/common/cpp/token_bucket_rate_limiter.cpp index 0e30f870de368e..128501ae7cf469 100644 --- a/common/cpp/token_bucket_rate_limiter.cpp +++ b/common/cpp/token_bucket_rate_limiter.cpp @@ -20,6 +20,7 @@ #include #include // IWYU pragma: export +#include #include #include #include @@ -32,6 +33,18 @@ namespace doris { // Just 10^6. static constexpr auto NS = 1000000000UL; +bvar::Adder s3_get_rate_limit_sleep_ns("s3_get_rate_limit_sleep_ns"); +bvar::Adder s3_get_rate_limit_sleep_count("s3_get_rate_limit_sleep_count"); +bvar::Adder s3_get_rate_limit_rejected_count("s3_get_rate_limit_rejected_count"); +bvar::Adder s3_put_rate_limit_sleep_ns("s3_put_rate_limit_sleep_ns"); +bvar::Adder s3_put_rate_limit_sleep_count("s3_put_rate_limit_sleep_count"); +bvar::Adder s3_put_rate_limit_rejected_count("s3_put_rate_limit_rejected_count"); + +static std::atomic s3_get_rate_limit_sleep_log_count {0}; +static std::atomic s3_get_rate_limit_rejected_log_count {0}; +static std::atomic s3_put_rate_limit_sleep_log_count {0}; +static std::atomic s3_put_rate_limit_rejected_log_count {0}; + class TokenBucketRateLimiter::SimpleSpinLock { public: SimpleSpinLock() = default; @@ -118,13 +131,20 @@ TokenBucketRateLimiterHolder::TokenBucketRateLimiterHolder(size_t max_speed, siz metric_func(std::move(metric_func)) {} int64_t TokenBucketRateLimiterHolder::add(size_t amount) { - int64_t sleep; + return add_with_config(amount).sleep_duration; +} + +TokenBucketRateLimiterResult TokenBucketRateLimiterHolder::add_with_config(size_t amount) { + TokenBucketRateLimiterResult result; { std::shared_lock read {rate_limiter_rw_lock}; - sleep = rate_limiter->add(amount); + result = {.sleep_duration = rate_limiter->add(amount), + .max_speed = rate_limiter->get_max_speed(), + .max_burst = rate_limiter->get_max_burst(), + .limit = rate_limiter->get_limit()}; } - metric_func(sleep); - return sleep; + metric_func(result.sleep_duration); + return result; } int TokenBucketRateLimiterHolder::reset(size_t max_speed, size_t max_burst, size_t limit) { @@ -135,6 +155,21 @@ int TokenBucketRateLimiterHolder::reset(size_t max_speed, size_t max_burst, size return 0; } +size_t TokenBucketRateLimiterHolder::get_max_speed() const { + std::shared_lock read {rate_limiter_rw_lock}; + return rate_limiter->get_max_speed(); +} + +size_t TokenBucketRateLimiterHolder::get_max_burst() const { + std::shared_lock read {rate_limiter_rw_lock}; + return rate_limiter->get_max_burst(); +} + +size_t TokenBucketRateLimiterHolder::get_limit() const { + std::shared_lock read {rate_limiter_rw_lock}; + return rate_limiter->get_limit(); +} + std::string to_string(S3RateLimitType type) { switch (type) { case S3RateLimitType::GET: @@ -154,4 +189,52 @@ S3RateLimitType string_to_s3_rate_limit_type(std::string_view value) { } return S3RateLimitType::UNKNOWN; } + +std::function s3_rate_limiter_metric_func(S3RateLimitType type) { + switch (type) { + case S3RateLimitType::GET: + return metric_func_factory(s3_get_rate_limit_sleep_ns, s3_get_rate_limit_sleep_count, + &s3_get_rate_limit_rejected_count); + case S3RateLimitType::PUT: + return metric_func_factory(s3_put_rate_limit_sleep_ns, s3_put_rate_limit_sleep_count, + &s3_put_rate_limit_rejected_count); + default: + return [](int64_t) {}; + } +} + +int64_t apply_s3_rate_limit(S3RateLimitType type, S3RateLimiterHolder* rate_limiter, + int64_t log_interval) { + auto result = rate_limiter->add_with_config(1); + auto sleep_duration = result.sleep_duration; + if (log_interval <= 0 || sleep_duration == 0) { + return sleep_duration; + } + + auto is_get = type == S3RateLimitType::GET; + auto* sleep_log_count = + is_get ? &s3_get_rate_limit_sleep_log_count : &s3_put_rate_limit_sleep_log_count; + auto* rejected_log_count = + is_get ? &s3_get_rate_limit_rejected_log_count : &s3_put_rate_limit_rejected_log_count; + + if (sleep_duration > 0) { + int64_t count = sleep_log_count->fetch_add(1, std::memory_order_relaxed) + 1; + if (count == 1 || count % log_interval == 0) { + LOG(INFO) << "S3 " << to_string(type) << " request is throttled by local rate limiter" + << ", sleep_ms=" << sleep_duration / 1000000 << ", sleep_count=" << count + << ", token_per_second=" << result.max_speed + << ", bucket_tokens=" << result.max_burst << ", token_limit=" << result.limit; + } + } else { + int64_t count = rejected_log_count->fetch_add(1, std::memory_order_relaxed) + 1; + if (count == 1 || count % log_interval == 0) { + LOG(WARNING) << "S3 " << to_string(type) << " request is rejected by local rate limiter" + << ", rejected_count=" << count + << ", token_per_second=" << result.max_speed + << ", bucket_tokens=" << result.max_burst + << ", token_limit=" << result.limit; + } + } + return sleep_duration; +} } // namespace doris diff --git a/common/cpp/token_bucket_rate_limiter.h b/common/cpp/token_bucket_rate_limiter.h index dbeca3b897db03..57f9205527acc6 100644 --- a/common/cpp/token_bucket_rate_limiter.h +++ b/common/cpp/token_bucket_rate_limiter.h @@ -21,6 +21,8 @@ #include #include #include +#include +#include namespace doris { enum class S3RateLimitType : int { @@ -32,11 +34,15 @@ enum class S3RateLimitType : int { extern std::string to_string(S3RateLimitType type); extern S3RateLimitType string_to_s3_rate_limit_type(std::string_view value); -inline auto metric_func_factory(bvar::Adder& ns_bvar, bvar::Adder& req_num_bvar) { - return [&](int64_t ns) { +inline auto metric_func_factory(bvar::Adder& sleep_ns_bvar, + bvar::Adder& sleep_count_bvar, + bvar::Adder* rejected_count_bvar = nullptr) { + return [&, rejected_count_bvar](int64_t ns) { if (ns > 0) { - ns_bvar << ns; - req_num_bvar << 1; + sleep_ns_bvar << ns; + sleep_count_bvar << 1; + } else if (ns < 0 && rejected_count_bvar != nullptr) { + *rejected_count_bvar << 1; } }; } @@ -71,6 +77,13 @@ class TokenBucketRateLimiter { long _prev_ns_count {0}; // Previous `add` call time (in nanoseconds). }; +struct TokenBucketRateLimiterResult { + int64_t sleep_duration; + size_t max_speed; + size_t max_burst; + size_t limit; +}; + class TokenBucketRateLimiterHolder { public: TokenBucketRateLimiterHolder(size_t max_speed, size_t max_burst, size_t limit, @@ -78,17 +91,16 @@ class TokenBucketRateLimiterHolder { ~TokenBucketRateLimiterHolder(); int64_t add(size_t amount); + TokenBucketRateLimiterResult add_with_config(size_t amount); int reset(size_t max_speed, size_t max_burst, size_t limit); - size_t get_max_speed() const { return rate_limiter->get_max_speed(); } - - size_t get_max_burst() const { return rate_limiter->get_max_burst(); } - - size_t get_limit() const { return rate_limiter->get_limit(); } + size_t get_max_speed() const; + size_t get_max_burst() const; + size_t get_limit() const; private: - std::shared_mutex rate_limiter_rw_lock; + mutable std::shared_mutex rate_limiter_rw_lock; std::unique_ptr rate_limiter; // Record the correspoding sleeping time(unit is ms) std::function metric_func; @@ -96,4 +108,8 @@ class TokenBucketRateLimiterHolder { using S3RateLimiter = TokenBucketRateLimiter; using S3RateLimiterHolder = TokenBucketRateLimiterHolder; -} // namespace doris \ No newline at end of file + +std::function s3_rate_limiter_metric_func(S3RateLimitType type); +int64_t apply_s3_rate_limit(S3RateLimitType type, S3RateLimiterHolder* rate_limiter, + int64_t log_interval); +} // namespace doris diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt index b0432407456d64..1b522d2df14fed 100644 --- a/dist/LICENSE-dist.txt +++ b/dist/LICENSE-dist.txt @@ -1476,6 +1476,9 @@ The Apache Software License, Version 2.0 * opentelemetry-proto: 0.18.0 * opentelemetry-cpp: 1.4.0 * clucene: 2.4.6 + * lance-c: 0.1.2 + - source: https://github.com/lance-format/lance-c + - transitive Rust crate licenses: licenses/LICENSE-lance-c-rust-crates.txt The MIT License -- licenses/LICENSE-MIT.txt * datatables: 1.12.1 diff --git a/dist/licenses/LICENSE-lance-c-rust-crates.txt b/dist/licenses/LICENSE-lance-c-rust-crates.txt new file mode 100644 index 00000000000000..41052e9b821920 --- /dev/null +++ b/dist/licenses/LICENSE-lance-c-rust-crates.txt @@ -0,0 +1,625 @@ +Lance-C Rust crate license list + +Generated from lance-c v0.1.2 Cargo.lock using: +cargo +1.91.0 tree --locked --edges normal,build + +This list excludes dev/test-only dependency edges. License values are SPDX +expressions from Cargo package metadata. + +(Apache-2.0 OR MIT) AND BSD-3-Clause + * encoding_rs: 0.8.35 + +(MIT OR Apache-2.0) AND Apache-2.0 + * moka: 0.12.15 + +(MIT OR Apache-2.0) AND Unicode-3.0 + * unicode-ident: 1.0.24 + +0BSD + * mock_instant: 0.6.0 + +0BSD OR MIT OR Apache-2.0 + * adler2: 2.0.1 + +Apache-2.0 + * approx: 0.5.1 + * arrow: 57.3.0 + * arrow-arith: 57.3.0 + * arrow-array: 57.3.0 + * arrow-buffer: 57.3.0 + * arrow-cast: 57.3.0 + * arrow-csv: 57.3.0 + * arrow-data: 57.3.0 + * arrow-ipc: 57.3.0 + * arrow-json: 57.3.0 + * arrow-ord: 57.3.0 + * arrow-row: 57.3.0 + * arrow-schema: 57.3.0 + * arrow-select: 57.3.0 + * arrow-string: 57.3.0 + * aws-config: 1.8.14 + * aws-credential-types: 1.2.13 + * aws-runtime: 1.7.1 + * aws-sdk-sso: 1.95.0 + * aws-sdk-ssooidc: 1.97.0 + * aws-sdk-sts: 1.99.0 + * aws-sigv4: 1.4.1 + * aws-smithy-async: 1.2.13 + * aws-smithy-http: 0.63.5 + * aws-smithy-http-client: 1.1.11 + * aws-smithy-json: 0.62.4 + * aws-smithy-observability: 0.2.5 + * aws-smithy-query: 0.60.14 + * aws-smithy-runtime: 1.10.2 + * aws-smithy-runtime-api: 1.11.5 + * aws-smithy-types: 1.4.5 + * aws-smithy-xml: 0.60.14 + * aws-types: 1.3.13 + * backon: 1.6.0 + * datafusion: 52.4.0 + * datafusion-catalog: 52.4.0 + * datafusion-catalog-listing: 52.4.0 + * datafusion-common: 52.4.0 + * datafusion-common-runtime: 52.4.0 + * datafusion-datasource: 52.4.0 + * datafusion-datasource-arrow: 52.4.0 + * datafusion-datasource-csv: 52.4.0 + * datafusion-datasource-json: 52.4.0 + * datafusion-datasource-parquet: 52.4.0 + * datafusion-doc: 52.4.0 + * datafusion-execution: 52.4.0 + * datafusion-expr: 52.4.0 + * datafusion-expr-common: 52.4.0 + * datafusion-functions: 52.4.0 + * datafusion-functions-aggregate: 52.4.0 + * datafusion-functions-aggregate-common: 52.4.0 + * datafusion-functions-nested: 52.4.0 + * datafusion-functions-table: 52.4.0 + * datafusion-functions-window: 52.4.0 + * datafusion-functions-window-common: 52.4.0 + * datafusion-macros: 52.4.0 + * datafusion-optimizer: 52.4.0 + * datafusion-physical-expr: 52.4.0 + * datafusion-physical-expr-adapter: 52.4.0 + * datafusion-physical-expr-common: 52.4.0 + * datafusion-physical-optimizer: 52.4.0 + * datafusion-physical-plan: 52.4.0 + * datafusion-pruning: 52.4.0 + * datafusion-session: 52.4.0 + * datafusion-sql: 52.4.0 + * datafusion-substrait: 52.4.0 + * flatbuffers: 25.12.19 + * fsst: 4.0.1 + * jsonb: 0.5.5 + * lance: 4.0.1 + * lance-arrow: 4.0.1 + * lance-bitpacking: 4.0.1 + * lance-c: 0.1.2 + * lance-core: 4.0.1 + * lance-datafusion: 4.0.1 + * lance-datagen: 4.0.1 + * lance-encoding: 4.0.1 + * lance-file: 4.0.1 + * lance-geo: 4.0.1 + * lance-index: 4.0.1 + * lance-io: 4.0.1 + * lance-linalg: 4.0.1 + * lance-namespace: 4.0.1 + * lance-namespace-reqwest-client: 0.6.1 + * lance-table: 4.0.1 + * object_store_opendal: 0.55.0 + * opendal: 0.55.0 + * parquet: 57.3.0 + * prost: 0.14.3 + * prost-build: 0.14.3 + * prost-derive: 0.14.3 + * prost-types: 0.14.3 + * reqsign: 0.16.5 + * serde_tokenstream: 0.2.3 + * sketches-ddsketch: 0.3.1 + * sqlparser: 0.59.0 + * sqlparser_derive: 0.3.0 + * substrait: 0.62.2 + * sync_wrapper: 1.0.2 + * thrift: 0.17.0 + * typify: 0.5.0 + * typify-impl: 0.5.0 + * typify-macro: 0.5.0 + +Apache-2.0 / MIT + * fnv: 1.0.7 + +Apache-2.0 / MIT / MPL-2.0 + * htmlescape: 0.3.1 + +Apache-2.0 AND ISC + * ring: 0.17.14 + +Apache-2.0 OR BSL-1.0 + * ryu: 1.0.23 + +Apache-2.0 OR ISC OR MIT + * hyper-rustls: 0.27.7 + * rustls: 0.23.37 + * rustls-native-certs: 0.8.3 + * rustls-pemfile: 2.2.0 + +Apache-2.0 OR MIT + * async-channel: 2.5.0 + * async-lock: 3.4.2 + * atomic-waker: 1.1.2 + * autocfg: 1.5.0 + * base64ct: 1.8.3 + * concurrent-queue: 2.5.0 + * const-oid: 0.9.6 + * der: 0.7.10 + * equivalent: 1.0.2 + * event-listener: 5.4.1 + * event-listener-strategy: 0.5.4 + * fastrand: 2.3.0 + * idna_adapter: 1.2.1 + * indexmap: 2.13.0 + * parking: 2.2.1 + * pem-rfc7468: 0.7.0 + * pin-project: 1.1.11 + * pin-project-internal: 1.1.11 + * pin-project-lite: 0.2.17 + * pkcs1: 0.7.5 + * pkcs5: 0.7.1 + * pkcs8: 0.10.2 + * portable-atomic: 1.13.1 + * rustc-hash: 2.1.1 + * signature: 2.2.0 + * spki: 0.7.3 + * utf8_iter: 1.0.4 + * uuid: 1.22.0 + * zeroize: 1.8.2 + +Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT + * linux-raw-sys: 0.12.1 + * linux-raw-sys: 0.4.15 + * rustix: 0.38.44 + * rustix: 1.1.4 + +Apache-2.0/MIT + * bytes-utils: 0.1.4 + * crc32c: 0.6.8 + * permutation: 0.4.1 + +BSD-2-Clause + * arrayref: 0.3.9 + +BSD-2-Clause OR Apache-2.0 OR MIT + * zerocopy: 0.8.47 + * zerocopy-derive: 0.8.47 + +BSD-3-Clause + * alloc-no-stdlib: 2.0.4 + * alloc-stdlib: 0.2.2 + * snap: 1.1.1 + * subtle: 2.6.1 + +BSD-3-Clause AND MIT + * brotli: 8.0.2 + +BSD-3-Clause OR MIT OR Apache-2.0 + * num_enum: 0.7.6 + * num_enum_derive: 0.7.6 + +BSD-3-Clause/MIT + * brotli-decompressor: 5.0.0 + +BSL-1.0 + * xxhash-rust: 0.8.15 + +CC0-1.0 + * tiny-keccak: 2.0.2 + +CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception + * blake3: 1.8.3 + +CC0-1.0 OR MIT-0 OR Apache-2.0 + * constant_time_eq: 0.4.2 + * dunce: 1.0.5 + +CDLA-Permissive-2.0 + * webpki-roots: 1.0.6 + +ISC + * earcutr: 0.4.3 + * rustls-webpki: 0.103.10 + * simple_asn1: 0.6.4 + * untrusted: 0.9.0 + +ISC AND (Apache-2.0 OR ISC) + * aws-lc-rs: 1.16.2 + +ISC AND (Apache-2.0 OR ISC) AND Apache-2.0 AND MIT AND BSD-3-Clause AND (Apache-2.0 OR ISC OR MIT) AND (Apache-2.0 OR ISC OR MIT-0) + * aws-lc-sys: 0.39.0 + +MIT + * async_cell: 0.2.3 + * atoi: 2.0.0 + * base64-simd: 0.8.0 + * bitpacking: 0.9.3 + * bitvec: 1.0.1 + * bytes: 1.11.1 + * census: 0.4.2 + * comfy-table: 7.2.2 + * crunchy: 0.2.4 + * darling: 0.23.0 + * darling_core: 0.23.0 + * darling_macro: 0.23.0 + * dashmap: 6.1.0 + * deepsize: 0.2.0 + * deepsize_derive: 0.1.2 + * float_next_after: 1.0.0 + * fs_extra: 1.3.0 + * funty: 2.0.0 + * generic-array: 0.14.7 + * geographiclib-rs: 0.2.7 + * h2: 0.4.13 + * http-body: 0.4.6 + * http-body: 1.0.1 + * http-body-util: 0.1.3 + * hyper: 1.8.1 + * hyper-util: 0.1.20 + * hyperloglogplus: 0.4.1 + * i_float: 1.15.0 + * i_key_sort: 0.6.0 + * i_overlay: 4.0.7 + * i_shape: 1.14.0 + * i_tree: 0.16.0 + * integer-encoding: 3.0.4 + * jsonwebtoken: 9.3.1 + * levenshtein_automata: 0.2.1 + * libm: 0.2.16 + * lru: 0.12.5 + * lz4: 1.28.1 + * lz4-sys: 1.11.1+lz4-1.10.0 + * lz4_flex: 0.11.6 + * lz4_flex: 0.12.1 + * measure_time: 0.9.0 + * mime_guess: 2.0.5 + * mio: 1.1.1 + * murmurhash32: 0.3.1 + * nom: 7.1.3 + * nom: 8.0.0 + * ordered-float: 2.10.1 + * ordered-float: 5.1.0 + * ordered-multimap: 0.7.3 + * outref: 0.5.2 + * ownedbytes: 0.9.0 + * pbjson: 0.8.0 + * pbjson-build: 0.8.0 + * pbjson-types: 0.8.0 + * pem: 3.0.6 + * phf: 0.12.1 + * phf_shared: 0.12.1 + * quick-xml: 0.37.5 + * quick-xml: 0.38.4 + * radium: 0.7.0 + * random_word: 0.5.2 + * rust-ini: 0.21.3 + * schemars: 0.8.22 + * schemars_derive: 0.8.22 + * simd-adler32: 0.3.8 + * slab: 0.4.12 + * spin: 0.9.8 + * std_prelude: 0.2.12 + * strsim: 0.11.1 + * strum: 0.26.3 + * strum_macros: 0.26.4 + * synstructure: 0.13.2 + * tantivy: 0.24.2 + * tantivy-bitpacker: 0.8.0 + * tantivy-columnar: 0.5.0 + * tantivy-common: 0.9.0 + * tantivy-query-grammar: 0.24.0 + * tantivy-sstable: 0.5.0 + * tantivy-stacker: 0.5.0 + * tantivy-tokenizer-api: 0.5.0 + * tap: 1.0.1 + * tokio: 1.50.0 + * tokio-macros: 2.6.1 + * tokio-stream: 0.1.18 + * tokio-util: 0.7.18 + * tower: 0.5.3 + * tower-http: 0.6.8 + * tower-layer: 0.3.3 + * tower-service: 0.3.3 + * tracing: 0.1.44 + * tracing-attributes: 0.1.31 + * tracing-core: 0.1.36 + * try-lock: 0.2.5 + * twox-hash: 2.1.2 + * unsafe-libyaml: 0.2.11 + * urlencoding: 2.1.3 + * vsimd: 0.8.0 + * want: 0.3.1 + * winnow: 1.0.0 + * wyz: 0.5.1 + * zmij: 1.0.21 + * zstd: 0.13.3 + +MIT OR Apache-2.0 + * aes: 0.8.4 + * ahash: 0.8.12 + * allocator-api2: 0.2.21 + * anyhow: 1.0.102 + * arc-swap: 1.9.0 + * arrayvec: 0.7.6 + * async-compression: 0.4.41 + * async-recursion: 1.1.1 + * async-trait: 0.1.89 + * base64: 0.22.1 + * bitflags: 2.11.0 + * blake2: 0.10.6 + * block-buffer: 0.10.4 + * block-padding: 0.3.3 + * bon: 3.9.1 + * bon-macros: 3.9.1 + * cbc: 0.1.2 + * cc: 1.2.57 + * cfg-if: 1.0.4 + * chrono: 0.4.44 + * chrono-tz: 0.10.4 + * cipher: 0.4.4 + * cmake: 0.1.57 + * compression-codecs: 0.4.37 + * compression-core: 0.4.31 + * const-random: 0.1.18 + * const-random-macro: 0.1.16 + * cpufeatures: 0.2.17 + * crc32fast: 1.5.0 + * crossbeam-channel: 0.5.15 + * crossbeam-deque: 0.8.6 + * crossbeam-epoch: 0.9.18 + * crossbeam-queue: 0.3.12 + * crossbeam-skiplist: 0.1.3 + * crossbeam-utils: 0.8.21 + * crypto-common: 0.1.7 + * deranged: 0.5.8 + * digest: 0.10.7 + * dirs: 6.0.0 + * dirs-sys: 0.5.0 + * displaydoc: 0.2.5 + * dlv-list: 0.5.2 + * downcast-rs: 2.0.2 + * dyn-clone: 1.0.20 + * either: 1.15.0 + * errno: 0.3.14 + * ethnum: 1.5.2 + * fast-float2: 0.2.3 + * find-msvc-tools: 0.1.9 + * fixedbitset: 0.5.7 + * flate2: 1.1.9 + * form_urlencoded: 1.2.2 + * fs4: 0.8.4 + * futures: 0.3.32 + * futures-channel: 0.3.32 + * futures-core: 0.3.32 + * futures-executor: 0.3.32 + * futures-io: 0.3.32 + * futures-macro: 0.3.32 + * futures-sink: 0.3.32 + * futures-task: 0.3.32 + * futures-util: 0.3.32 + * geo: 0.31.0 + * geo-traits: 0.3.0 + * geo-types: 0.7.18 + * geoarrow-array: 0.7.0 + * geoarrow-expr-geo: 0.7.0 + * geoarrow-schema: 0.7.0 + * geodatafusion: 0.3.0 + * getrandom: 0.2.17 + * getrandom: 0.3.4 + * getrandom: 0.4.2 + * glob: 0.3.3 + * half: 2.7.1 + * hash32: 0.3.1 + * hashbrown: 0.14.5 + * hashbrown: 0.15.5 + * hashbrown: 0.16.1 + * heapless: 0.8.0 + * heck: 0.5.0 + * hex: 0.4.3 + * hmac: 0.12.1 + * home: 0.5.12 + * http: 0.2.12 + * http: 1.4.0 + * httparse: 1.10.1 + * humantime: 2.3.0 + * iana-time-zone: 0.1.65 + * idna: 1.1.0 + * inout: 0.1.4 + * ipnet: 2.12.0 + * iri-string: 0.7.11 + * itertools: 0.11.0 + * itertools: 0.13.0 + * itertools: 0.14.0 + * itoa: 1.0.18 + * jobserver: 0.1.34 + * lazy_static: 1.5.0 + * libc: 0.2.183 + * lock_api: 0.4.14 + * log: 0.4.29 + * md-5: 0.10.6 + * memmap2: 0.9.10 + * mime: 0.3.17 + * multimap: 0.10.1 + * ndarray: 0.16.1 + * num-bigint: 0.4.6 + * num-complex: 0.4.6 + * num-conv: 0.2.0 + * num-integer: 0.1.46 + * num-iter: 0.1.45 + * num-traits: 0.2.19 + * num_cpus: 1.17.0 + * once_cell: 1.21.4 + * oneshot: 0.1.13 + * openssl-probe: 0.2.1 + * parking_lot: 0.12.5 + * parking_lot_core: 0.9.12 + * paste: 1.0.15 + * path_abs: 0.5.1 + * pbkdf2: 0.12.2 + * percent-encoding: 2.3.2 + * petgraph: 0.8.3 + * pin-utils: 0.1.0 + * pkg-config: 0.3.32 + * powerfmt: 0.2.0 + * ppv-lite86: 0.2.21 + * prettyplease: 0.2.37 + * proc-macro-crate: 3.5.0 + * proc-macro2: 1.0.106 + * quote: 1.0.45 + * rand: 0.8.5 + * rand: 0.9.2 + * rand_chacha: 0.3.1 + * rand_chacha: 0.9.0 + * rand_core: 0.6.4 + * rand_core: 0.9.5 + * rand_distr: 0.4.3 + * rand_distr: 0.5.1 + * rand_xoshiro: 0.7.0 + * rayon: 1.11.0 + * rayon-core: 1.13.0 + * regex: 1.12.3 + * regex-automata: 0.4.14 + * regex-lite: 0.1.9 + * regex-syntax: 0.8.10 + * regress: 0.10.5 + * reqwest: 0.12.28 + * roaring: 0.11.3 + * robust: 1.2.0 + * rsa: 0.9.10 + * rstar: 0.12.2 + * rustc_version: 0.4.1 + * rustls-pki-types: 1.14.0 + * rustversion: 1.0.22 + * salsa20: 0.10.2 + * scopeguard: 1.2.0 + * scrypt: 0.11.0 + * semver: 1.0.27 + * seq-macro: 0.3.6 + * serde: 1.0.228 + * serde_core: 1.0.228 + * serde_derive: 1.0.228 + * serde_derive_internals: 0.29.1 + * serde_json: 1.0.149 + * serde_repr: 0.1.20 + * serde_yaml: 0.9.34+deprecated + * sha1: 0.10.6 + * sha2: 0.10.9 + * shlex: 1.3.0 + * signal-hook-registry: 1.4.8 + * simdutf8: 0.1.5 + * smallvec: 1.15.1 + * snafu: 0.9.0 + * snafu-derive: 0.9.0 + * socket2: 0.6.3 + * spade: 2.15.0 + * stable_deref_trait: 1.2.1 + * stfu8: 0.2.7 + * syn: 1.0.109 + * syn: 2.0.117 + * tempfile: 3.27.0 + * thiserror: 1.0.69 + * thiserror: 2.0.18 + * thiserror-impl: 1.0.69 + * thiserror-impl: 2.0.18 + * thread-tree: 0.3.3 + * time: 0.3.47 + * time-core: 0.1.8 + * time-macros: 0.2.27 + * tokio-rustls: 0.26.4 + * toml_datetime: 1.1.0+spec-1.1.0 + * toml_edit: 0.25.8+spec-1.1.0 + * toml_parser: 1.1.0+spec-1.1.0 + * typenum: 1.19.0 + * unicase: 2.9.0 + * unicode-segmentation: 1.13.0 + * unicode-width: 0.2.2 + * url: 2.5.8 + * wkb: 0.9.2 + * wkt: 0.14.0 + * zstd-safe: 7.2.4 + +MIT OR Zlib OR Apache-2.0 + * miniz_oxide: 0.8.9 + +MIT/Apache-2.0 + * bigdecimal: 0.4.10 + * geohash: 0.13.1 + * ident_case: 1.0.1 + * lexical-core: 1.0.6 + * lexical-parse-float: 1.0.6 + * lexical-parse-integer: 1.0.6 + * lexical-util: 1.0.7 + * lexical-write-float: 1.0.6 + * lexical-write-integer: 1.0.6 + * matrixmultiply: 0.3.10 + * minimal-lexical: 0.2.1 + * num-bigint-dig: 0.8.6 + * object_store: 0.12.5 + * rangemap: 1.7.1 + * rawpointer: 0.2.1 + * serde_urlencoded: 0.7.1 + * siphasher: 1.0.2 + * tagptr: 0.2.0 + * version_check: 0.9.5 + * xmlparser: 0.13.6 + * zstd-sys: 2.0.16+zstd.1.5.7 + +MIT/BSD-3-Clause + * rust-stemmers: 1.2.0 + +MPL-2.0 + * option-ext: 0.2.0 + +Unicode-3.0 + * icu_collections: 2.1.1 + * icu_locale_core: 2.1.1 + * icu_normalizer: 2.1.1 + * icu_normalizer_data: 2.1.1 + * icu_properties: 2.1.2 + * icu_properties_data: 2.1.2 + * icu_provider: 2.1.1 + * litemap: 0.8.1 + * potential_utf: 0.1.4 + * tinystr: 0.8.2 + * writeable: 0.6.2 + * yoke: 0.8.1 + * yoke-derive: 0.8.1 + * zerofrom: 0.1.6 + * zerofrom-derive: 0.1.6 + * zerotrie: 0.2.3 + * zerovec: 0.11.5 + * zerovec-derive: 0.11.2 + +Unlicense OR MIT + * aho-corasick: 1.1.4 + * byteorder: 1.5.0 + * jiff: 0.2.23 + * memchr: 2.8.0 + +Unlicense/MIT + * csv: 1.4.0 + * csv-core: 0.1.13 + * fst: 0.4.7 + * same-file: 1.0.6 + * tantivy-fst: 0.5.0 + * utf8-ranges: 1.0.5 + * walkdir: 2.5.0 + +Zlib + * foldhash: 0.1.5 + * foldhash: 0.2.0 + * zlib-rs: 0.6.3 + +Zlib OR Apache-2.0 OR MIT + * bytemuck: 1.25.0 + +zlib-acknowledgement OR MIT + * fastdivide: 0.4.2 diff --git a/docker/runtime/doris-compose/requirements.txt b/docker/runtime/doris-compose/requirements.txt index e519f99af032ea..930ba78eb4049b 100644 --- a/docker/runtime/doris-compose/requirements.txt +++ b/docker/runtime/doris-compose/requirements.txt @@ -22,7 +22,7 @@ jsonpickle prettytable pymysql python-dateutil -requests<=2.31.0 +requests>=2.31.0 # NOTICE: if install docker failed, specific pyyaml version and docker version #pyyaml==5.3.1 diff --git a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql index 039ddd1d3690fb..d45b878d1ed29d 100644 --- a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql @@ -342,6 +342,14 @@ CREATE TABLE doris_test.extreme_test ) ENGINE = MergeTree() ORDER BY id; +CREATE TABLE doris_test.distributed_type AS doris_test.type +ENGINE = Distributed(default, doris_test, type, rand()); + +CREATE MATERIALIZED VIEW doris_test.materialized_view_type +ENGINE = MergeTree +ORDER BY k1 +AS SELECT * FROM doris_test.type; + CREATE TABLE doris_test.extreme_test_multi_block ( id UInt64, @@ -392,4 +400,4 @@ CREATE TABLE doris_test.extreme_test_multi_block ipv6_col IPv6, ipv6_nullable Nullable(IPv6) ) ENGINE = MergeTree() -ORDER BY id; \ No newline at end of file +ORDER BY id; diff --git a/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl b/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl index 7c5dba3910925b..565ac5ae4d4a31 100644 --- a/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl +++ b/docker/thirdparties/docker-compose/iceberg/entrypoint.sh.tpl @@ -48,21 +48,21 @@ start-history-server.sh # This approach can reduce the time from 150s to 40s. START_TIME1=$(date +%s) -find /mnt/scripts/create_preinstalled_scripts/iceberg -name '*.sql' | sed 's|^|source |' | sed 's|$|;|'> iceberg_total.sql +find /mnt/scripts/create_preinstalled_scripts/iceberg -name '*.sql' | sort | sed 's|^|source |' | sed 's|$|;|'> iceberg_total.sql spark-sql --master spark://doris--spark-iceberg:7077 --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions -f iceberg_total.sql END_TIME1=$(date +%s) EXECUTION_TIME1=$((END_TIME1 - START_TIME1)) echo "Script iceberg total: {} executed in $EXECUTION_TIME1 seconds" START_TIME2=$(date +%s) -find /mnt/scripts/create_preinstalled_scripts/paimon -name '*.sql' | sed 's|^|source |' | sed 's|$|;|'> paimon_total.sql +find /mnt/scripts/create_preinstalled_scripts/paimon -name '*.sql' | sort | sed 's|^|source |' | sed 's|$|;|'> paimon_total.sql spark-sql --master spark://doris--spark-iceberg:7077 --conf spark.sql.extensions=org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions -f paimon_total.sql END_TIME2=$(date +%s) EXECUTION_TIME2=$((END_TIME2 - START_TIME2)) echo "Script paimon total: {} executed in $EXECUTION_TIME2 seconds" START_TIME3=$(date +%s) -find /mnt/scripts/create_preinstalled_scripts/iceberg_load -name '*.sql' | sed 's|^|source |' | sed 's|$|;|'> iceberg_load_total.sql +find /mnt/scripts/create_preinstalled_scripts/iceberg_load -name '*.sql' | sort | sed 's|^|source |' | sed 's|$|;|'> iceberg_load_total.sql spark-sql --master spark://doris--spark-iceberg:7077 --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions -f iceberg_load_total.sql END_TIME3=$(date +%s) EXECUTION_TIME3=$((END_TIME3 - START_TIME3)) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run31.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run31.sql new file mode 100644 index 00000000000000..b1cf79240bb657 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run31.sql @@ -0,0 +1,33 @@ +-- Bootstrap an Iceberg table whose active snapshot contains both Parquet and ORC data files. +-- This script is sourced on every Iceberg container start, so keep it repeatable. + +CREATE DATABASE IF NOT EXISTS demo.test_db; +USE demo.test_db; + +DROP TABLE IF EXISTS mixed_file_format; + +CREATE TABLE mixed_file_format ( + id INT, + source STRING +) +USING iceberg +TBLPROPERTIES ( + 'format-version' = '2', + 'write.format.default' = 'parquet' +); + +-- The first snapshot's data files are Parquet. +INSERT INTO mixed_file_format VALUES + (1, 'parquet'), + (2, 'parquet'), + (3, 'parquet'); + +-- Change only the format for subsequent writes. The Parquet files above remain +-- referenced by the current snapshot, while this append produces ORC files. +ALTER TABLE mixed_file_format + SET TBLPROPERTIES ('write.format.default' = 'orc'); + +INSERT INTO mixed_file_format VALUES + (4, 'orc'), + (5, 'orc'), + (6, 'orc'); diff --git a/docker/thirdparties/docker-compose/kerberos/.gitignore b/docker/thirdparties/docker-compose/kerberos/.gitignore new file mode 100644 index 00000000000000..c7e96926b15ca5 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/.gitignore @@ -0,0 +1,8 @@ +# Generated by run-thirdparties-docker.sh. +/conf/kerberos1/*.conf +/conf/kerberos1/*.xml +/conf/kerberos2/*.conf +/conf/kerberos2/*.xml +/data/ +/two-kerberos-hives/*.conf +/two-kerberos-hives/*.keytab diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh b/docker/thirdparties/docker-compose/kerberos/Dockerfile old mode 100755 new mode 100644 similarity index 61% rename from docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh rename to docker/thirdparties/docker-compose/kerberos/Dockerfile index 77df431d85ac3b..88723e1313b999 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh +++ b/docker/thirdparties/docker-compose/kerberos/Dockerfile @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,25 +15,20 @@ # specific language governing permissions and limitations # under the License. -set -euo pipefail +FROM apache/hive:3.1.3 -if test $# -gt 0; then - echo "$0 does not accept arguments" >&2 - exit 32 -fi +USER root -# Supervisord is not running -if ! test -f /tmp/supervisor.sock; then - exit 0 -fi +RUN apt-get -o Acquire::Retries=3 -o APT::Update::Error-Mode=any update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + krb5-admin-server \ + krb5-kdc \ + krb5-user \ + && rm -rf /var/lib/apt/lists/* -# Check if all Hadoop services are running -FAILED=$(supervisorctl status | grep -v RUNNING || true) +COPY entrypoint-hive-master.sh /opt/doris/entrypoint.sh +COPY health-checks/health.sh /opt/doris/health.sh -if [ "$FAILED" == "" ]; then - echo "All services are running" - exit 0 -else - echo "Some of the services are failing: ${FAILED}" - exit 1 -fi +RUN chmod 755 /opt/doris/entrypoint.sh /opt/doris/health.sh + +ENTRYPOINT ["/opt/doris/entrypoint.sh"] diff --git a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf index 83fe29c2cb2fa0..c7d41612a5359e 100644 --- a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf +++ b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf @@ -33,15 +33,9 @@ [realms] LABS.TERADATA.COM = { kdc = hadoop-master:5588 - admin_server = hadoop-master:5749 - } - OTHERLABS.TERADATA.COM = { - kdc = hadoop-master:5589 - admin_server = hadoop-master:5750 } OTHERREALM.COM = { kdc = hadoop-master-2:6688 - admin_server = hadoop-master-2:6749 } [domain_realm] diff --git a/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh b/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh deleted file mode 100755 index ec2dc074e705cd..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# test whether OVERRIDES_DIR is set -if [[ -n "${OVERRIDES_DIR+x}" ]]; then - echo "The OVERRIDES_DIR (${OVERRIDES_DIR}) support is disabled as it was deemed unused." >&2 - echo "It is being removed." >&2 - exit 16 -fi - -if test -e /overrides; then - find /overrides >&2 - echo "The /overrides handling is disabled as it was deemed unused." >&2 - echo "It is being removed." >&2 - exit 17 -fi diff --git a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh b/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh deleted file mode 100755 index 93c6e385effe1f..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -if test $# -gt 0; then - echo "$0 does not accept arguments" >&2 - exit 32 -fi - -set -x - -HADOOP_INIT_D=${HADOOP_INIT_D:-/etc/hadoop-init.d/} - -echo "Applying hadoop init.d scripts from ${HADOOP_INIT_D}" -if test -d "${HADOOP_INIT_D}"; then - for init_script in "${HADOOP_INIT_D}"*; do - chmod a+x "${init_script}" - "${init_script}" - done -fi - -trap exit INT - -echo "Running services with supervisord" -rm -rf /etc/supervisord.d/socks-proxy.conf -rm -rf /etc/supervisord.d/sshd.conf - -supervisord -c /etc/supervisord.conf diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml b/docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl old mode 100755 new mode 100644 similarity index 58% rename from docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml rename to docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl index c0ce38e3cdc1bd..c5625a53ef639d --- a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml +++ b/docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl @@ -1,4 +1,4 @@ - + + + fs.defaultFS + hdfs://${HOST}:${FS_PORT} + + + hadoop.security.authentication + kerberos + + + hadoop.security.authorization + true + hadoop.security.auth_to_local - RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*// + RULE:[2:$1@$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:$1@$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*// DEFAULT + + hadoop.proxyuser.hive.hosts + * + + + hadoop.proxyuser.hive.users + * + + + hadoop.security.token.service.use_ip + false + diff --git a/docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl b/docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl new file mode 100644 index 00000000000000..b56a18f5ca57d4 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl @@ -0,0 +1,113 @@ + + + + + dfs.namenode.name.dir + file:///data/hdfs/name + + + dfs.datanode.data.dir + file:///data/hdfs/data + + + dfs.replication + 1 + + + dfs.permissions.enabled + false + + + dfs.namenode.datanode.registration.ip-hostname-check + false + + + dfs.block.access.token.enable + true + + + dfs.namenode.kerberos.principal + hdfs/${HOST}@${REALM} + + + dfs.namenode.rpc-bind-host + 0.0.0.0 + + + dfs.namenode.keytab.file + /data/keytabs/hdfs.keytab + + + dfs.datanode.kerberos.principal + hdfs/${HOST}@${REALM} + + + dfs.datanode.keytab.file + /data/keytabs/hdfs.keytab + + + dfs.namenode.kerberos.internal.spnego.principal + HTTP/${HOST}@${REALM} + + + dfs.web.authentication.kerberos.principal + HTTP/${HOST}@${REALM} + + + dfs.web.authentication.kerberos.keytab + /data/keytabs/spnego.keytab + + + dfs.data.transfer.protection + authentication + + + dfs.http.policy + HTTP_ONLY + + + ignore.secure.ports.for.testing + true + + + dfs.datanode.address + 0.0.0.0:${DFS_DN_PORT} + + + dfs.datanode.http.address + 0.0.0.0:${DFS_DN_HTTP_PORT} + + + dfs.datanode.ipc.address + 0.0.0.0:${DFS_DN_IPC_PORT} + + + dfs.datanode.hostname + ${HOST} + + + dfs.client.use.datanode.hostname + true + + + dfs.namenode.http-address + 0.0.0.0:${DFS_NN_HTTP_PORT} + + diff --git a/docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl b/docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl new file mode 100644 index 00000000000000..2b4b41e20a7341 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl @@ -0,0 +1,65 @@ + + + + + javax.jdo.option.ConnectionURL + jdbc:derby:;databaseName=/data/metastore/metastore_db;create=true + + + javax.jdo.option.ConnectionDriverName + org.apache.derby.jdbc.EmbeddedDriver + + + datanucleus.autoCreateSchema + false + + + hive.metastore.schema.verification + true + + + hive.metastore.uris + thrift://${HOST}:${HMS_PORT} + + + hive.metastore.warehouse.dir + hdfs://${HOST}:${FS_PORT}/user/hive/warehouse + + + hive.metastore.sasl.enabled + true + + + hive.metastore.kerberos.principal + hive/${HOST}@${REALM} + + + hive.metastore.kerberos.keytab.file + /data/keytabs/hive.keytab + + + hive.metastore.execute.setugi + true + + + metastore.storage.schema.reader.impl + org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader + + diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh b/docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl old mode 100755 new mode 100644 similarity index 74% rename from docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh rename to docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl index ab464b5233b31a..0b678db1538447 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh +++ b/docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,5 +15,15 @@ # specific language governing permissions and limitations # under the License. -kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM -beeline -u "jdbc:hive2://localhost:15000/default;principal=hive/hadoop-master@LABS.TERADATA.COM" -e "show databases;" \ No newline at end of file +[kdcdefaults] + kdc_ports = ${KDC_PORT} + kdc_tcp_ports = ${KDC_PORT} + +[realms] + ${REALM} = { + database_name = /data/kdc/principal + key_stash_file = /data/kdc/.k5.${REALM} + max_life = 24h + max_renewable_life = 7d + supported_enctypes = aes128-cts-hmac-sha1-96:normal + } diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl deleted file mode 100644 index e16c70e16db817..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[kdcdefaults] - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - -[realms] - LABS.TERADATA.COM = { - acl_file = /var/kerberos/krb5kdc/kadm5.acl - dict_file = /usr/share/dict/words - admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab - supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal - kdc_listen = ${KDC_PORT1} - kdc_tcp_listen = ${KDC_PORT1} - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - } - - OTHERLABS.TERADATA.COM = { - acl_file = /var/kerberos/krb5kdc/kadm5-other.acl - dict_file = /usr/share/dict/words - admin_keytab = /var/kerberos/krb5kdc/kadm5-other.keytab - supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal - kdc_listen = ${KDC_PORT2} - kdc_tcp_listen = ${KDC_PORT2} - kdc_ports = ${KDC_PORT2} - kdc_tcp_ports = ${KDC_PORT2} - kadmind_port = ${KADMIND_PORT2} - kpasswd_port = ${KPASSWD_PORT2} - } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl deleted file mode 100644 index 1edf2bb8fd05fe..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[logging] - default = FILE:/var/log/krb5libs.log - kdc = FILE:/var/log/krb5kdc.log - admin_server = FILE:/var/log/kadmind.log - -[libdefaults] - default_realm = LABS.TERADATA.COM - dns_lookup_realm = false - dns_lookup_kdc = false - forwardable = true - allow_weak_crypto = true - -[realms] - LABS.TERADATA.COM = { - kdc = ${HOST}:${KDC_PORT1} - admin_server = ${HOST}:${KADMIND_PORT1} - } - OTHERLABS.TERADATA.COM = { - kdc = ${HOST}:${KDC_PORT2} - admin_server = ${HOST}:${KADMIND_PORT2} - } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl deleted file mode 100644 index 61b4994ad5c1ac..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[kdcdefaults] - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - - -[realms] - OTHERREALM.COM = { - acl_file = /var/kerberos/krb5kdc/kadm5.acl - dict_file = /usr/share/dict/words - admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab - supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal - kdc_listen = ${KDC_PORT1} - kdc_tcp_listen = ${KDC_PORT1} - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/krb5.conf.tpl similarity index 81% rename from docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl rename to docker/thirdparties/docker-compose/kerberos/conf/krb5.conf.tpl index c817dbdd79753d..84d8707184e1bc 100644 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl +++ b/docker/thirdparties/docker-compose/kerberos/conf/krb5.conf.tpl @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -19,17 +18,18 @@ [logging] default = FILE:/var/log/krb5libs.log kdc = FILE:/var/log/krb5kdc.log - admin_server = FILE:/var/log/kadmind.log [libdefaults] - default_realm = OTHERREALM.COM + default_realm = ${REALM} dns_lookup_realm = false dns_lookup_kdc = false forwardable = true - allow_weak_crypto = true + udp_preference_limit = 1 [realms] - OTHERREALM.COM = { - kdc = ${HOST}:${KDC_PORT1} - admin_server = ${HOST}:${KADMIND_PORT1} - } \ No newline at end of file + ${REALM} = { + kdc = 127.0.0.1:${KDC_PORT} + } + +[domain_realm] + ${HOST} = ${REALM} diff --git a/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl deleted file mode 100644 index e91c65c100463f..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[mysqld] -port=${MYSQL_PORT} -datadir=/var/lib/mysql -socket=/var/lib/mysql/mysql.sock -# Disabling symbolic-links is recommended to prevent assorted security risks -symbolic-links=0 -# Settings user and group are ignored when systemd is used. -# If you need to run mysqld under a different user or group, -# customize your systemd unit file for mariadb according to the -# instructions in http://fedoraproject.org/wiki/Systemd - -[mysqld_safe] -log-error=/var/log/mariadb/mariadb.log -pid-file=/var/run/mariadb/mariadb.pid - -# -# include all files from the config directory -# -!includedir /etc/my.cnf.d - diff --git a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh index 16b0cad0283950..5dd4adf46d458e 100755 --- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh +++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh @@ -15,83 +15,125 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + set -euo pipefail -source /usr/local/common/hive-configure.sh -source /usr/local/common/event-hook.sh - -echo "Configuring hive" -configure /etc/hive/conf/hive-site.xml hive HIVE_SITE_CONF -configure /etc/hive/conf/hiveserver2-site.xml hive HIVE_SITE_CONF -configure /etc/hadoop/conf/core-site.xml core CORE_CONF -configure /etc/hadoop/conf/hdfs-site.xml hdfs HDFS_CONF -configure /etc/hadoop/conf/yarn-site.xml yarn YARN_CONF -configure /etc/hadoop/conf/mapred-site.xml mapred MAPRED_CONF -configure /etc/hive/conf/beeline-site.xml beeline BEELINE_SITE_CONF - -echo "Copying kerberos keytabs to keytabs/" -mkdir -p /etc/hadoop-init.d/ - -if [ "$1" == "1" ]; then - cp /etc/trino/conf/* /keytabs/ -elif [ "$1" == "2" ]; then - cp /etc/trino/conf/hive-presto-master.keytab /keytabs/other-hive-presto-master.keytab - cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab -else - echo "Invalid index parameter. Exiting." - exit 1 -fi -cd /usr/hdp/3.1.0.0-78/hive/auxlib -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jdom-1.1.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-core-3.4.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ecs-4.2.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ram-3.0.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-sts-3.0.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jindo-core-6.3.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jindo-core-linux-el7-aarch64-6.3.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jindo-sdk-6.3.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aws-java-sdk-bundle-1.11.375.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-huaweicloud-3.1.1-hw-54.5.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-cos-3.1.0-8.3.22.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/cos_api-bundle-5.6.244.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-aws-3.2.1.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/paimon-hive-connector-3.1-1.3-SNAPSHOT.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/gcs-connector-hadoop3-2.2.24-shaded.jar - - -/usr/local/hadoop-run.sh & - -# check healthy hear -echo "Waiting for hadoop to be healthy" - -for i in {1..60}; do - if /usr/local/health.sh; then - echo "Hadoop is healthy" + +: "${HOST:?}" +: "${REALM:?}" +: "${KDC_PORT:?}" +: "${FS_PORT:?}" +: "${HMS_PORT:?}" +: "${HIVE_CLIENT_KEYTAB:?}" +: "${PRESTO_CLIENT_KEYTAB:?}" + +export HADOOP_CONF_DIR=/opt/doris/conf +export HIVE_CONF_DIR=/opt/doris/conf +export KRB5_KDC_PROFILE=/opt/doris/conf/kdc.conf +export PATH="/usr/local/sbin:/usr/sbin:${PATH}" +export HADOOP_CLIENT_OPTS="-Xms192m -Xmx320m" + +readonly HDFS_PRINCIPAL="hdfs/${HOST}@${REALM}" +readonly HTTP_PRINCIPAL="HTTP/${HOST}@${REALM}" +readonly HIVE_PRINCIPAL="hive/${HOST}@${REALM}" +readonly HIVE_CLIENT_PRINCIPAL="hive/presto-master.docker.cluster@${REALM}" +readonly PRESTO_CLIENT_PRINCIPAL="presto-server/presto-master.docker.cluster@${REALM}" + +declare -a SERVICE_PIDS=() + +stop_services() { + kill "${SERVICE_PIDS[@]}" 2>/dev/null || true + wait "${SERVICE_PIDS[@]}" 2>/dev/null || true +} + +trap stop_services EXIT INT TERM + +start_service() { + "$@" & + SERVICE_PIDS+=("$!") +} + +wait_for_port() { + local host=$1 + local port=$2 + local service=$3 + + for _ in {1..120}; do + if (exec 3<>"/dev/tcp/${host}/${port}") 2>/dev/null; then + exec 3>&- + exec 3<&- + return + fi + sleep 1 + done + + echo "Timed out waiting for ${service} on ${host}:${port}" >&2 + return 1 +} + +create_keytab() { + local principal=$1 + local keytab=$2 + + kadmin.local -r "${REALM}" -q "addprinc -randkey ${principal}" + kadmin.local -r "${REALM}" -q "ktadd -k ${keytab} ${principal}" +} + +report_stage() { + echo "DORIS_KERBEROS_STAGE=$1" +} + +report_stage "prepare-data" +mkdir -p /data/hdfs/data /data/hdfs/name /data/kdc /data/keytabs /data/metastore /keytabs +rm -rf /tmp/hadoop-server-conf +cp -R "${HADOOP_CONF_DIR}" /tmp/hadoop-server-conf +sed -i "s#hdfs://${HOST}:${FS_PORT}#hdfs://127.0.0.1:${FS_PORT}#" \ + /tmp/hadoop-server-conf/core-site.xml + +report_stage "initialize-kdc" +kdb5_util create -s -r "${REALM}" -P doris-kerberos-test +create_keytab "${HDFS_PRINCIPAL}" /data/keytabs/hdfs.keytab +create_keytab "${HTTP_PRINCIPAL}" /data/keytabs/spnego.keytab +create_keytab "${HIVE_PRINCIPAL}" /data/keytabs/hive.keytab +create_keytab "${HIVE_CLIENT_PRINCIPAL}" "/keytabs/${HIVE_CLIENT_KEYTAB}" +create_keytab "${PRESTO_CLIENT_PRINCIPAL}" "/keytabs/${PRESTO_CLIENT_KEYTAB}" +chmod 644 /keytabs/*.keytab + +report_stage "start-kdc" +start_service krb5kdc -n -r "${REALM}" +wait_for_port 127.0.0.1 "${KDC_PORT}" "Kerberos KDC" + +report_stage "start-namenode" +export HDFS_NAMENODE_OPTS="-Xms128m -Xmx256m" +export HDFS_DATANODE_OPTS="-Xms96m -Xmx192m" +hdfs namenode -format -force -nonInteractive +start_service hdfs namenode +wait_for_port "${HOST}" "${FS_PORT}" "HDFS NameNode" + +report_stage "start-datanode" +start_service env HADOOP_CONF_DIR=/tmp/hadoop-server-conf hdfs datanode +wait_for_port "${HOST}" "${DFS_DN_PORT}" "HDFS DataNode" + +report_stage "wait-for-datanode-registration" +export KRB5CCNAME=FILE:/tmp/hdfs-admin.ccache +kinit -kt /data/keytabs/hdfs.keytab "${HDFS_PRINCIPAL}" +for _ in {1..120}; do + if hdfs dfsadmin -Dfs.defaultFS="hdfs://127.0.0.1:${FS_PORT}" -report 2>/dev/null \ + | grep -q '^Live datanodes (1):'; then break fi - echo "Hadoop is not healthy yet. Retrying in 60 seconds..." - sleep 5 + sleep 1 done +hdfs dfsadmin -Dfs.defaultFS="hdfs://127.0.0.1:${FS_PORT}" -report \ + | grep -q '^Live datanodes (1):' +kdestroy + +report_stage "initialize-hive-metastore" +schematool -dbType derby -initSchema +start_service hive --service metastore -p "${HMS_PORT}" +wait_for_port "${HOST}" "${HMS_PORT}" "Hive Metastore" + +touch /tmp/SUCCESS +echo "Minimal Kerberos HDFS and Hive Metastore environment is ready" +echo "DORIS_KERBEROS_READY" -if [ $i -eq 60 ]; then - echo "Hadoop did not become healthy after 60 attempts. Exiting." - exit 1 -fi - -echo "Init kerberos test data" - -if [ "$1" == "1" ]; then - kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM -elif [ "$1" == "2" ]; then - kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM -else - echo "Invalid index parameter. Exiting." - exit 1 -fi -hive -f /usr/local/sql/create_kerberos_hive_table.sql -if [[ ${enablePaimonHms} == "true" ]]; then - echo "Creating Paimon HMS catalog and table" - hadoop fs -put /tmp/paimon_data/* /user/hive/warehouse/ - hive -f /usr/local/sql/create_paimon_hive_table.hql -fi - -exec_success_hook \ No newline at end of file +wait -n "${SERVICE_PIDS[@]}" diff --git a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl index 5afa33af88dae2..792d0c0491da97 100644 --- a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl +++ b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl @@ -1,95 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://127.0.0.1:${MYSQL_PORT}/metastore -HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.jdbc.Driver -HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=root -HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=root -HIVE_SITE_CONF_datanucleus_autoCreateSchema=false -HIVE_SITE_CONF_hive_metastore_port=${HMS_PORT} -HIVE_SITE_CONF_hive_metastore_uris=thrift://${IP_HOST}:${HMS_PORT} -HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0 -HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT} -HIVE_SITE_CONF_hive_server2_webui_port=0 -HIVE_SITE_CONF_hive_compactor_initiator_on=true -HIVE_SITE_CONF_hive_compactor_worker_threads=2 -HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader -BEELINE_SITE_CONF_beeline_hs2_jdbc_url_tcpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive -BEELINE_SITE_CONF_beeline_hs2_jdbc_url_httpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive - - -CORE_CONF_fs_defaultFS=hdfs://${HOST}:${FS_PORT} -CORE_CONF_hadoop_http_staticuser_user=root -CORE_CONF_hadoop_proxyuser_hue_hosts=* -CORE_CONF_hadoop_proxyuser_hue_groups=* - -HDFS_CONF_dfs_webhdfs_enabled=true -HDFS_CONF_dfs_permissions_enabled=false -HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false -HDFS_CONF_dfs_datanode_address=${HOST}:${DFS_DN_PORT} -HDFS_CONF_dfs_datanode_http_address=${HOST}:${DFS_DN_HTTP_PORT} -HDFS_CONF_dfs_datanode_ipc_address=${HOST}:${DFS_DN_IPC_PORT} -HDFS_CONF_dfs_namenode_http___address=${HOST}:${DFS_NN_HTTP_PORT} -YARN_CONF_yarn_log___aggregation___enable=true -YARN_CONF_yarn_resourcemanager_recovery_enabled=true -YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore -YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate -YARN_CONF_yarn_nodemanager_remote___app___log___dir=/var/log/hadoop-yarn/apps -YARN_CONF_yarn_log_server_url=http://${HOST}:${YARM_LOG_SERVER_PORT}/jobhistory/logs -YARN_CONF_yarn_timeline___service_enabled=false -YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true -YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true -YARN_CONF_yarn_resourcemanager_hostname=${HOST} -MAPRED_CONF_mapreduce_shuffle_port=${MAPREDUCE_SHUFFLE_PORT} -YARN_CONF_yarn_timeline___service_hostname=${HOST} -YARN_CONF_yarn_resourcemanager_address=${HOST}:${YARN_RM_PORT} -YARN_CONF_yarn_resourcemanager_scheduler_address=${HOST}:${YARN_RM_SCHEDULER_PORT} -YARN_CONF_yarn_resourcemanager_resource___tracker_address=${HOST}:${YARN_RM_TRACKER_PORT} -YARN_CONF_yarn_resourcemanager_admin_address=${HOST}:${YARN_RM_ADMIN_PORT} -YARN_CONF_yarn_resourcemanager_webapp_address=${HOST}:${YARN_RM_WEBAPP_PORT} -YARN_CONF_yarn_nodemanager_localizer_address=${HOST}:${YARN_NM_LOCAL_PORT} -YARN_CONF_yarn_nodemanager_webapp_address=${HOST}:${YARN_NM_WEBAPP_PORT} +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. -HIVE_SITE_CONF_fs_s3_impl=org.apache.hadoop.fs.s3a.S3AFileSystem -HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem -HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk} -HIVE_SITE_CONF_fs_s3a_secret_key=${AWSSk} -HIVE_SITE_CONF_fs_s3a_endpoint=${AWSEndpoint} -HIVE_SITE_CONF_fs_AbstractFileSystem_obs_impl=org.apache.hadoop.fs.obs.OBS -HIVE_SITE_CONF_fs_obs_access_key=${OBSAk} -HIVE_SITE_CONF_fs_obs_secret_key=${OBSSk} -HIVE_SITE_CONF_fs_obs_endpoint=${OBSEndpoint} -HIVE_SITE_CONF_fs_cosn_credentials_provider=org.apache.hadoop.fs.auth.SimpleCredentialProvider -HIVE_SITE_CONF_fs_cosn_userinfo_secretId=${COSAk} -HIVE_SITE_CONF_fs_cosn_userinfo_secretKey=${COSSk} -HIVE_SITE_CONF_fs_cosn_bucket_region=${COSRegion} -HIVE_SITE_CONF_fs_cosn_impl=org.apache.hadoop.fs.CosFileSystem -HIVE_SITE_CONF_fs_cos_impl=org.apache.hadoop.fs.CosFileSystem -HIVE_SITE_CONF_fs_AbstractFileSystem_cosn_impl=org.apache.hadoop.fs.CosN -HIVE_SITE_CONF_fs_oss_impl=com.aliyun.jindodata.oss.JindoOssFileSystem -HIVE_SITE_CONF_fs_AbstractFileSystem_oss_impl=com.aliyun.jindodata.oss.JindoOSS -HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk} -HIVE_SITE_CONF_fs_oss_accessKeySecret=${OSSSk} -HIVE_SITE_CONF_fs_oss_endpoint=${OSSEndpoint} -HIVE_SITE_CONF_fs_AbstractFileSystem_gs_impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS -HIVE_SITE_CONF_fs_gs_project_id=${GCSProjectId} -HIVE_SITE_CONF_google_cloud_auth_service_account_enable=true -HIVE_SITE_CONF_fs_gs_auth_service_account_email=${GCSAccountEmail} -HIVE_SITE_CONF_fs_gs_auth_service_account_private_key_id=${GCSAccountPrivateKeyId} -HIVE_SITE_CONF_fs_gs_auth_service_account_private_key=${GCSAccountPrivateKey} -HIVE_SITE_CONF_fs_gs_proxy_address=${GCSProxyAddress} -enablePaimonHms=${enablePaimonHms} \ No newline at end of file +HOST=${HOST} +REALM=${REALM} +KDC_PORT=${KDC_PORT} +FS_PORT=${FS_PORT} +HMS_PORT=${HMS_PORT} +DFS_DN_PORT=${DFS_DN_PORT} +DFS_DN_HTTP_PORT=${DFS_DN_HTTP_PORT} +DFS_DN_IPC_PORT=${DFS_DN_IPC_PORT} +DFS_NN_HTTP_PORT=${DFS_NN_HTTP_PORT} +HIVE_CLIENT_KEYTAB=${HIVE_CLIENT_KEYTAB} +PRESTO_CLIENT_KEYTAB=${PRESTO_CLIENT_KEYTAB} +HADOOP_CONF_DIR=/opt/doris/conf +HIVE_CONF_DIR=/opt/doris/conf +KRB5_CONFIG=/etc/krb5.conf diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh index 515f37e36ac9e3..07cd7edb379f3b 100755 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh +++ b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh @@ -18,17 +18,11 @@ set -euo pipefail -if test $# -gt 0; then - echo "$0 does not accept arguments" >&2 - exit 32 -fi - -set -x - -HEALTH_D=${HEALTH_D:-/etc/health.d/} - -if test -d "${HEALTH_D}"; then - for health_script in "${HEALTH_D}"/*; do - "${health_script}" &>> /var/log/container-health.log || exit 1 - done -fi +test -f /tmp/SUCCESS +kinit -kt /data/keytabs/hive.keytab "hive/${HOST}@${REALM}" +exec 3<>"/dev/tcp/${HOST}/${FS_PORT}" +exec 3>&- +exec 3<&- +exec 3<>"/dev/tcp/${HOST}/${HMS_PORT}" +exec 3>&- +exec 3<&- diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh deleted file mode 100755 index 7545969bc47d65..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM -beeline -u "jdbc:hive2://localhost:16000/default;principal=hive/hadoop-master-2@OTHERREALM.COM" -e "show databases;" \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl index c253c06f3e488b..d21496516afac4 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl +++ b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl @@ -14,59 +14,41 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -version: "3" + +x-kerberos-image: &kerberos-image + image: doris-kerberos-minimal:${CONTAINER_UID} + build: + context: . + dockerfile: Dockerfile + network: host + +x-kerberos-service: &kerberos-service + <<: *kerberos-image + network_mode: "host" + mem_limit: 1536m + stop_grace_period: 30s + services: hive-krb1: - image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized_96 + <<: *kerberos-service container_name: doris-${CONTAINER_UID}-kerberos1 + hostname: hadoop-master volumes: - - ../common:/usr/local/common + - ./conf/kerberos1:/opt/doris/conf:ro + - ./conf/kerberos1/krb5.conf:/etc/krb5.conf:ro + - ./data/kerberos1:/data - ./two-kerberos-hives:/keytabs - - ./sql:/usr/local/sql - - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh - - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh - - ./health-checks/health.sh:/usr/local/health.sh - - ./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh - - ./health-checks/hive-health-check.sh:/etc/health.d/hive-health-check.sh - - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh - - ./conf/kerberos1/my.cnf:/etc/my.cnf - - ./conf/kerberos1/kdc.conf:/var/kerberos/krb5kdc/kdc.conf - - ./conf/kerberos1/krb5.conf:/etc/krb5.conf - - ./paimon_data:/tmp/paimon_data - hostname: hadoop-master - entrypoint: /usr/local/entrypoint-hive-master.sh 1 - healthcheck: - test: ["CMD", "ls", "/tmp/SUCCESS"] - interval: 5s - timeout: 10s - retries: 120 - network_mode: "host" env_file: - ./hadoop-hive-1.env + hive-krb2: - image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized-2_96 + <<: *kerberos-service container_name: doris-${CONTAINER_UID}-kerberos2 hostname: hadoop-master-2 volumes: - - ../common:/usr/local/common + - ./conf/kerberos2:/opt/doris/conf:ro + - ./conf/kerberos2/krb5.conf:/etc/krb5.conf:ro + - ./data/kerberos2:/data - ./two-kerberos-hives:/keytabs - - ./sql:/usr/local/sql - - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh - - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh - - ./health-checks/health.sh:/usr/local/health.sh - - ./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh - - ./health-checks/hive-health-check-2.sh:/etc/health.d/hive-health-check-2.sh - - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh - - ./conf/kerberos2/my.cnf:/etc/my.cnf - - ./conf/kerberos2/kdc.conf:/var/kerberos/krb5kdc/kdc.conf - - ./conf/kerberos2/krb5.conf:/etc/krb5.conf - - ./paimon_data:/tmp/paimon_data - entrypoint: /usr/local/entrypoint-hive-master.sh 2 - healthcheck: - test: ["CMD", "ls", "/tmp/SUCCESS"] - interval: 5s - timeout: 10s - retries: 120 - network_mode: "host" env_file: - - ./hadoop-hive-2.env \ No newline at end of file + - ./hadoop-hive-2.env diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env index 44f2352cf16bfb..0e019dfcdd9f18 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env +++ b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,53 +16,14 @@ # specific language governing permissions and limitations # under the License. -# Change this to a specific string. -# Do not use "_" or other sepcial characters, only number and alphabeta. -# NOTICE: change this uid will modify hive-*.yaml - export HOST="hadoop-master" +export REALM="LABS.TERADATA.COM" +export KDC_PORT=5588 export FS_PORT=8520 export HMS_PORT=9583 -export HS_PORT=15000 -export MYSQL_PORT=3356 export DFS_DN_PORT=9566 export DFS_DN_HTTP_PORT=9564 export DFS_DN_IPC_PORT=9567 export DFS_NN_HTTP_PORT=9570 -export YARM_LOG_SERVER_PORT=8588 -export YARN_RM_PORT=8532 -export YARN_RM_SCHEDULER_PORT=8530 -export YARN_RM_TRACKER_PORT=8531 -export YARN_RM_ADMIN_PORT=8533 -export YARN_RM_WEBAPP_PORT=8589 -export YARN_NM_LOCAL_PORT=8540 -export YARN_NM_WEBAPP_PORT=8542 -export MAPREDUCE_SHUFFLE_PORT=13562 -export KADMIND_PORT=5464 -export KDC_PORT1=5588 -export KDC_PORT2=5589 -export KADMIND_PORT1=5749 -export KADMIND_PORT2=5750 -export KPASSWD_PORT1=5464 -export KPASSWD_PORT2=5465 - -# prepare for paimon hms test,control load paimon hms data or not -export enablePaimonHms="false" -# hms on s3/oss/obs/cos -export AWSAk="*****************" -export AWSSk="*****************" -export AWSEndpoint="s3.ap-northeast-1.amazonaws.com" -export OSSAk="*****************" -export OSSSk="*****************" -export OSSEndpoint="oss-cn-beijing.aliyuncs.com" -export OBSAk="*****************" -export OBSSk="*****************" -export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com" -export COSAk="*****************" -export COSSk="*****************" -export COSRegion="ap-beijing" -export GCSProjectId="" -export GCSAccountEmail="" -export GCSAccountPrivateKeyId="" -export GCSAccountPrivateKey="" -export GCSProxyAddress="" \ No newline at end of file +export HIVE_CLIENT_KEYTAB="hive-presto-master.keytab" +export PRESTO_CLIENT_KEYTAB="presto-server.keytab" diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env index 49099e0f44163f..a41c7f7ccc137b 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env +++ b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,49 +16,14 @@ # specific language governing permissions and limitations # under the License. -# Change this to a specific string. -# Do not use "_" or other sepcial characters, only number and alphabeta. -# NOTICE: change this uid will modify hive-*.yaml - export HOST="hadoop-master-2" +export REALM="OTHERREALM.COM" +export KDC_PORT=6688 export FS_PORT=8620 export HMS_PORT=9683 -export HS_PORT=16000 -export MYSQL_PORT=3366 export DFS_DN_PORT=9666 export DFS_DN_HTTP_PORT=9664 export DFS_DN_IPC_PORT=9667 export DFS_NN_HTTP_PORT=9670 -export YARM_LOG_SERVER_PORT=8688 -export YARN_RM_PORT=8632 -export YARN_RM_SCHEDULER_PORT=8630 -export YARN_RM_TRACKER_PORT=8631 -export YARN_RM_ADMIN_PORT=8633 -export YARN_RM_WEBAPP_PORT=8689 -export YARN_NM_LOCAL_PORT=8640 -export YARN_NM_WEBAPP_PORT=8642 -export MAPREDUCE_SHUFFLE_PORT=13662 -export KDC_PORT1=6688 -export KADMIND_PORT1=6749 -export KPASSWD_PORT1=6464 - -# prepare for paimon hms test,control load paimon hms data or not -export enablePaimonHms="false" -# hms on s3/oss/obs/cos -export AWSAk="*****************" -export AWSSk="*****************" -export AWSEndpoint="s3.ap-northeast-1.amazonaws.com" -export OSSAk="*****************" -export OSSSk="*****************" -export OSSEndpoint="oss-cn-beijing.aliyuncs.com" -export OBSAk="*****************" -export OBSSk="*****************" -export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com" -export COSAk="*****************" -export COSSk="*****************" -export COSRegion="ap-beijing" -export GCSProjectId="" -export GCSAccountEmail="" -export GCSAccountPrivateKeyId="" -export GCSAccountPrivateKey="" -export GCSProxyAddress="" \ No newline at end of file +export HIVE_CLIENT_KEYTAB="other-hive-presto-master.keytab" +export PRESTO_CLIENT_KEYTAB="other-presto-server.keytab" diff --git a/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql b/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql deleted file mode 100644 index ecf58e88158b12..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `test_krb_hive_db`; -CREATE TABLE IF NOT EXISTS `test_krb_hive_db`.`test_krb_hive_tbl`( - `id_key` int, - `string_key` string, - `rate_val` double, - `comment` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; - -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(1, 'a', 3.16, 'cc0'); -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(2, 'b', 41.2, 'cc1'); -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(3, 'c', 6.2, 'cc2'); -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(4, 'd', 1.4, 'cc3'); diff --git a/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql b/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql deleted file mode 100644 index 65f463fef22360..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE DATABASE IF NOT EXISTS hdfs_db; - -USE hdfs_db; - -CREATE EXTERNAL TABLE external_test_table - STORED BY 'org.apache.paimon.hive.PaimonStorageHandler' -LOCATION 'hdfs:///user/hive/warehouse/hdfs_db.db/external_test_table'; \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml b/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml deleted file mode 100755 index 4541c1328aea16..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - fs.default.name - hdfs://hadoop-master-2:9000 - - diff --git a/docker/thirdparties/docker-compose/oceanbase/oceanbase.env b/docker/thirdparties/docker-compose/oceanbase/oceanbase.env index 5a8998e7cd5cff..108843f9686855 100644 --- a/docker/thirdparties/docker-compose/oceanbase/oceanbase.env +++ b/docker/thirdparties/docker-compose/oceanbase/oceanbase.env @@ -17,3 +17,4 @@ # under the License. DOCKER_OCEANBASE_EXTERNAL_PORT=2881 +DOCKER_OCEANBASE_PROXY_EXTERNAL_PORT=2883 diff --git a/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl b/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl index aa64c1d2144b5c..fee16e3c1600d5 100644 --- a/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl +++ b/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl @@ -19,16 +19,18 @@ version: "2.1" services: doris--oceanbase: - image: oceanbase/oceanbase-ce:4.2.1-lts + image: quay.io/oceanbase/obbinlog-ce:4.2.5-test restart: always environment: MODE: slim OB_MEMORY_LIMIT: 5G + PASSWORD: 123456 TZ: Asia/Shanghai ports: - ${DOCKER_OCEANBASE_EXTERNAL_PORT}:2881 + - ${DOCKER_OCEANBASE_PROXY_EXTERNAL_PORT}:2883 healthcheck: - test: ["CMD-SHELL", "obclient -h127.1 -P2881 -uroot@test -e 'SELECT * FROM doris_test.all_types limit 1'"] + test: ["CMD-SHELL", "obclient -h127.0.0.1 -P2881 -uroot@test -p123456 -e 'SELECT * FROM doris_test.all_types LIMIT 1' >/dev/null && obclient -h127.0.0.1 -P2883 -uroot@test -p123456 -Nse 'SHOW MASTER STATUS' | grep -q ."] interval: 5s timeout: 60s retries: 120 diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 26965ce24bb9dd..9585d45149b834 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -1472,18 +1472,110 @@ start_lakesoul() { fi } +dump_kerberos_container_state() { + local container="$1" + + echo "===== ${container} state =====" >&2 + sudo docker inspect --format \ + 'status={{.State.Status}} running={{.State.Running}} exitCode={{.State.ExitCode}} oomKilled={{.State.OOMKilled}} error={{.State.Error}}' \ + "${container}" >&2 || true + echo "===== ${container} logs (tail -200) =====" >&2 + sudo docker logs --tail 200 "${container}" >&2 || true +} + +cleanup_kerberos_ready_wait() { + local wait_pid="$1" + + [[ -n "${wait_pid}" ]] || return 0 + sudo kill -TERM -- "-${wait_pid}" >/dev/null 2>&1 || true + wait "${wait_pid}" >/dev/null 2>&1 || true +} + +wait_for_kerberos_ready() { + local container="$1" + local wait_pid="" + local wait_status=0 + + echo "Waiting for ${container} readiness event" + setsid sudo timeout --signal=TERM --kill-after=5s 20m bash -c ' + log_dir=$(mktemp -d) + mkfifo "${log_dir}/container.log" + docker logs --follow "$1" >"${log_dir}/container.log" 2>&1 & + log_pid=$! + cleanup() { + kill -KILL "${log_pid}" 2>/dev/null || true + wait "${log_pid}" 2>/dev/null || true + rm -rf "${log_dir}" + } + trap cleanup EXIT + while IFS= read -r line; do + if [[ "${line}" == "DORIS_KERBEROS_READY" ]]; then + echo "${line}" + exit 0 + fi + done <"${log_dir}/container.log" + exit 1 + ' _ "${container}" & + wait_pid=$! + trap 'cleanup_kerberos_ready_wait "${wait_pid}"' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + if ! wait "${wait_pid}"; then + wait_status=1 + fi + wait_pid="" + trap - EXIT TERM INT + if [[ "${wait_status}" -ne 0 ]]; then + echo "ERROR: timed out or container exited before ${container} became ready" >&2 + dump_kerberos_container_state "${container}" + return 1 + fi +} + +cleanup_kerberos_readiness_jobs() { + local pid + + for pid in "$@"; do + kill "${pid}" >/dev/null 2>&1 || true + done + for pid in "$@"; do + wait "${pid}" >/dev/null 2>&1 || true + done +} + +validate_kerberos_container() { + local container="$1" + + echo "Running one-shot readiness validation for ${container}" + if ! sudo docker exec "${container}" /opt/doris/health.sh; then + echo "ERROR: one-shot readiness validation failed for ${container}" >&2 + dump_kerberos_container_state "${container}" + return 1 + fi +} + start_kerberos() { echo "RUN_KERBEROS" + local KERBEROS_DIR="${ROOT}/docker-compose/kerberos" + local -a containers=( + "doris-${CONTAINER_UID}-kerberos1" + "doris-${CONTAINER_UID}-kerberos2" + ) + local -a readiness_pids=() + local readiness_status=0 + local container + local pid + export CONTAINER_UID=${CONTAINER_UID} - envsubst <"${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl >"${ROOT}"/docker-compose/kerberos/kerberos.yaml - sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/kerberos/entrypoint-hive-master.sh - sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/kerberos/entrypoint-hive-master.sh + envsubst <"${KERBEROS_DIR}/kerberos.yaml.tpl" >"${KERBEROS_DIR}/kerberos.yaml" + mkdir -p "${KERBEROS_DIR}/conf/kerberos1" "${KERBEROS_DIR}/conf/kerberos2" \ + "${KERBEROS_DIR}/two-kerberos-hives" for i in {1..2}; do - . "${ROOT}"/docker-compose/kerberos/kerberos${i}_settings.env - envsubst <"${ROOT}"/docker-compose/kerberos/hadoop-hive.env.tpl >"${ROOT}"/docker-compose/kerberos/hadoop-hive-${i}.env - envsubst <"${ROOT}"/docker-compose/kerberos/conf/my.cnf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/my.cnf - envsubst <"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf - envsubst <"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf + . "${KERBEROS_DIR}/kerberos${i}_settings.env" + envsubst <"${KERBEROS_DIR}/hadoop-hive.env.tpl" >"${KERBEROS_DIR}/hadoop-hive-${i}.env" + for config in kdc.conf krb5.conf core-site.xml hdfs-site.xml hive-site.xml; do + envsubst <"${KERBEROS_DIR}/conf/${config}.tpl" >"${KERBEROS_DIR}/conf/kerberos${i}/${config}" + done done sudo chmod a+w /etc/hosts if ! awk -v ip="${IP_HOST}" '$1 == ip && $2 == "hadoop-master" { found = 1 } END { exit !found }' /etc/hosts; then @@ -1492,18 +1584,38 @@ start_kerberos() { if ! awk -v ip="${IP_HOST}" '$1 == ip && $2 == "hadoop-master-2" { found = 1 } END { exit !found }' /etc/hosts; then sudo sed -i "1i${IP_HOST} hadoop-master-2" /etc/hosts fi - register_stack_metadata "kerberos" "${ROOT}/docker-compose/kerberos/kerberos.yaml" "" - compose_cmd "${ROOT}/docker-compose/kerberos/kerberos.yaml" "" down --remove-orphans - sudo rm -rf "${ROOT}"/docker-compose/kerberos/data + register_stack_metadata "kerberos" "${KERBEROS_DIR}/kerberos.yaml" "" + compose_cmd "${KERBEROS_DIR}/kerberos.yaml" "" down --remove-orphans + sudo rm -rf "${KERBEROS_DIR}/data" if [[ "${STOP}" -ne 1 ]]; then echo "PREPARE KERBEROS DATA" - rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.keytab - rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.jks - rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.conf - compose_cmd "${ROOT}/docker-compose/kerberos/kerberos.yaml" "" up --remove-orphans --wait -d - sudo ln -sfn "${ROOT}/docker-compose/kerberos/two-kerberos-hives" /keytabs - sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /keytabs/krb5.conf - sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /etc/krb5.conf + rm -rf "${KERBEROS_DIR}"/two-kerberos-hives/*.keytab + rm -rf "${KERBEROS_DIR}"/two-kerberos-hives/*.jks + rm -rf "${KERBEROS_DIR}"/two-kerberos-hives/*.conf + compose_cmd "${KERBEROS_DIR}/kerberos.yaml" "" up --build --remove-orphans -d + trap 'cleanup_kerberos_readiness_jobs "${readiness_pids[@]}"' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + for container in "${containers[@]}"; do + wait_for_kerberos_ready "${container}" & + readiness_pids+=("$!") + done + for pid in "${readiness_pids[@]}"; do + if ! wait "${pid}"; then + readiness_status=1 + fi + done + readiness_pids=() + trap - EXIT TERM INT + if [[ "${readiness_status}" -ne 0 ]]; then + return 1 + fi + for container in "${containers[@]}"; do + validate_kerberos_container "${container}" + done + sudo ln -sfn "${KERBEROS_DIR}/two-kerberos-hives" /keytabs + sudo cp "${KERBEROS_DIR}/common/conf/doris-krb5.conf" /keytabs/krb5.conf + sudo cp "${KERBEROS_DIR}/common/conf/doris-krb5.conf" /etc/krb5.conf sleep 2 fi } diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index b05ea798ee46b0..82a3ba03667753 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -99,14 +99,135 @@ format-specific checklist when reviewing Parquet or ORC. - Keep index construction, predicate translation, cache lookup, and virtual-column setup out of per-row and repeated batch paths unless the work is inherently row-local. Avoid repeated schema traversal, expression cloning, metadata parsing, allocation, and conversion. +- Keep Profile counters in the visible `FileScannerV2 -> TableReader -> FileReader -> IO` hierarchy. + Format-specific readers, such as `ParquetReader`, belong below `FileReader`; lifecycle, metadata, + index, predicate, decode, materialization, and physical I/O paths must all have timers at the layer + that owns the work. Flush recursively aggregated child-reader statistics at every batch boundary, + including empty-selection and error exits, so a slow in-progress scan is diagnosable before close. - Require format readers to populate the common `ReaderStatistics` accurately where applicable: filtered/read row groups, Bloom and min/max pruning, filtered group/page/lazy rows, read rows and bytes, metadata/footer/cache timing, page-index work, predicate time, dictionary rewrite, and Bloom read time. +- Reject dead or ambiguous counters. In particular, `FilteredBytes` counts compressed bytes of + projected physical chunks avoided by pruning, not every child in the Row Group; footer read, + footer parse, lazy page-index materialization, and page-index predicate evaluation need distinct + timers. Raw I/O counters stay under `IO` even when a format reader initiates the request. - Evaluate performance with representative format versions, writers, data ordering, predicate selectivity, nested width, remote storage, batch sizes, and warm/cold caches. Report both the optimization overhead and the avoided work; a low pruning ratio alone is not a defect. +## Parquet Native Decode Boundary + +- V2 must instantiate only readers and decoders under `be/src/format_v2/parquet/`; calls into the + v1 `ParquetColumnReader` or edits under `be/src/format/parquet/` are review blockers. +- Footer parsing, schema-ID assignment, and the cached native metadata tree must also be v2-owned. + Reusing a stable base file identity is allowed, but require a v2 cache type discriminator so + v1/v2 metadata objects can never be cross-cast. Production planning must not retain or rebuild an + Arrow `FileMetaData` tree from the serialized footer. +- Trace the hot path as `ColumnReader -> Decoder span/cursor API -> DataTypeSerDe -> Doris Column`. + Decoder must not accept a Doris column or target type, and the path must not create Arrow arrays, + builders, `DecodedColumnView`, or another decoded leaf batch. +- Verify physical/logical metadata is immutable per leaf reader and complete for signed integers, + decimal precision/scale, date/time/timestamp units and UTC adjustment, INT96, UUID, FLOAT16, and + fixed-width binary. Unsupported combinations return explicit errors before plausible output. +- Treat legacy Parquet `TIMESTAMP_MILLIS` and `TIMESTAMP_MICROS` converted types as UTC-adjusted. + Do not give them the local/unspecified semantics of an unannotated INT64 timestamp; data decode, + statistics conversion, and min/max pruning must use the same timezone rule. +- Route plain, dictionary, and decoded timestamp inputs through one checked conversion contract. + Validate INT96 nanos-of-day before widened Julian-day arithmetic, reject unit scaling overflow, + and enforce Doris year 0001-9999 before materialization. Conversion failures must follow the same + strict/non-strict and dictionary-ID propagation rules as other direct types. +- Verify schema-change routing separately from physical decode. The reader must emit the projected + file type, while `ColumnMapper`/`TableReader` perform file-to-table casts after file predicates. + Any different requested type at the native reader boundary must fail as an invariant violation; + do not add a reader-local conversion column or decoder-facing conversion ABI. +- Dictionary review must separate dictionary-entry IDs from logical rows and non-null payload + ordinals. Materialize the typed dictionary once per generation through the same SerDe, validate + every index before access, and invalidate cached dictionary state at Row Group/file/type changes. + Dictionary-entry predicate evaluation and later row-value flattening must reuse that same typed + generation rather than serializing, parsing, or converting the dictionary twice. +- Check direct materialization for PLAIN, RLE/dictionary, DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Filtering must advance encoded + values without allocating output; null runs must append defaults without advancing payload. +- For predicate-only fixed-width PLAIN primitives, allow direct comparison only after proving the + whole Column Chunk uses compatible PLAIN value pages and every Expr advertises raw fixed-value + evaluation with identical Doris comparison semantics, including NaN ordering. The fallback + decision must precede definition-level consumption. Disable the direct path when a residual or + delete conjunct still references the hidden slot, because it needs the materialized payload. + Verify sparse input selection, interleaved NULLs, reversed literal comparisons, multiple ANDed + comparisons, mixed-encoding fallback, residual slot reuse, and a stable row-shaped placeholder. +- For a filtered scalar page fragment, require one SerDe entry and one batch-level selected-decode + dispatch. Nullable selections must first map logical rows to selected non-NULL physical ranges, + decode those ranges once, and restore NULL slots in place; falling back per NULL run is a review + blocker for a scalar destination that supports in-place expansion. Selection ranges belong to + persistent reader scratch; per-range virtual SerDe/decoder calls in the hot path are a review + blocker. Fixed PLAIN should + bulk-gather spans, BYTE_ARRAY PLAIN should scan lengths once, dictionary decode should validate + every ID before gathering selected IDs, and stateful encodings should batch-decode/reconstruct and + compact. Any remaining NULL-interleaving fallback must preserve logical output order without a + decoded intermediate column and be counted by `HybridSelectionNullFallbackBatches`. +- Review complex types as a level/shape problem around scalar leaf materialization. Parent offsets, + null maps, sibling alignment, page-spanning rows, and child payload counts must remain correct + without materializing an intermediate complex column. +- For MAP, the non-null key leaf owns the outer entry shape. Validate the materialized key/value + entry counts, but do not compare their raw repetition vectors: a nested value legitimately has + deeper levels. For STRUCT, compare each sibling only at the current parent boundary and ignore + repetition owned by a deeper child collection. +- Require a bounded high-water policy for persistent definition/repetition, null, selection, + conversion, dictionary-index, and decoder-owned scratch. Distinguish active bytes from retained + capacity: never release an oversized buffer while the current batch still needs it, and require + three ordinary/idle batches before releasing capacity above the high-water limit. Test both + outlier release and steady-state reuse so the policy does not create allocation thrash. +- Review all decoder read and skip paths as equally exposed corruption boundaries. Check requested + counts against remaining/declared values before pointer arithmetic or narrowing; use checked + addition/multiplication for byte extents; bound BYTE_ARRAY dictionary entry counts and IDs before + allocation/indexing; and require DELTA_BYTE_ARRAY prefixes to fit the previous reconstructed + value. BOOLEAN RLE and DELTA skip paths must consume bounded chunks and fail on short streams. +- Validate every decoded definition/repetition level against the schema maximum in batch, run, and + single-value cursor APIs. Page value counts must not drive eager nested scratch allocation; + reserve from the requested parent-row frontier and cover tiny-payload huge-count Page V1/V2 files. +- Before Snappy decompression, inspect the encoded uncompressed length and validate destination + capacity. Page V1, Page V2, and dictionary pages must produce exactly the declared decoded size; + an UNCOMPRESSED dictionary page must also declare equal compressed and uncompressed sizes. +- For a STRUCT whose projected children are all missing after schema evolution, require a + levels-only physical reference leaf. It must advance and validate encoded payload cursors while + deriving the synthetic child count, without constructing a discarded string/complex column. +- `CountColumnReader` must use the native levels-only reader and must not decode payload or call + Arrow `ReadRecords`. Require profiles that distinguish page I/O, decompression, level decode, + value decode, SerDe materialization, hybrid selection batches/ranges/NULL fallback, + filtered-value skips, and page fragmentation. +- Validate every signed Column Chunk offset/length before converting it to `size_t`. The dictionary + offset is usable only when it is non-negative and precedes the data offset; the complete range + must fit the file. Apply the PARQUET-816 tail padding only to affected parquet-mr versions, cap it + at 100 bytes, and keep it inside the file. Scalar and levels-only COUNT readers share this helper. +- Treat OffsetIndex as one optional, all-or-nothing navigation structure. Require first row zero, + the first physical location to equal the owning ColumnMetaData `data_page_offset`, strictly + increasing row ordinals and physical offsets, positive sizes, non-overlapping page ranges, and + containment in the owning Column Chunk. Discard a malformed index before selecting the indexed + reader. +- Page iteration skips `INDEX_PAGE` and unknown auxiliary pages before initializing a data decoder. + Dictionary pages retain their special first-page handling; a later dictionary page is corrupt. +- Derive writer workarounds once from `created_by` and pass them through scalar, nested, page-cache, + and COUNT paths. Pre-Arrow-3 parquet-cpp Data Page V2 payloads remain compressed despite the + historical `is_compressed=false` flag. +- Preserve nullable conversion semantics in direct native materialization. Numeric, DATE, + DATETIME, TIME, and DECIMAL failures insert a default nested value and mark the corresponding NULL + only in non-strict mode; strict or non-nullable reads return the error. Dictionary failures follow + the selected dictionary IDs to output rows. +- Keep Parquet decimals in a source-width or wider intermediate until exact scaling, target + precision, and overflow checks succeed. Scale-down with a non-zero remainder is a conversion + failure; plain and dictionary integer/binary paths must narrow only afterward. +- For cold small-file tests, separate footer I/O/Thrift parse from native schema and index planning. + V2 must not retain serialized footer bytes after the native metadata tree is initialized; v1 opens + remain independent. +- For HTTP Parquet objects at or below `in_memory_file_size`, v2 stages the complete object from + byte zero before native page access. Verify both cold and footer-cache-hit scans: some Range + servers accept the capability probe but return HTTP 200 for a near-EOF overlong range, so warm + scans must not depend on the footer read having populated an incidental transport buffer. +- Identical fixed-width POD values append with one bulk copy. FIXED_LEN_BYTE_ARRAY strings copy the + dense byte span once and synthesize offsets; validate this execution contract without flaky + wall-clock assertions. + ## Parquet Multi-Level Filtering - Use [FileScannerV2 Parquet Scan Design](file-scanner-v2-parquet-scan-design.md) as the detailed @@ -129,12 +250,37 @@ format-specific checklist when reviewing Parquet or ORC. - Verify lazy materialization avoids reading and decoding non-predicate columns for rejected rows while advancing all readers correctly. Predicate columns should be read/prefetched first; output prefetch should wait for survivors when filtering is active. +- For safe staged single-column predicates, review the observed cost/rejection ordering and its + cold-start behavior. Reordering is allowed only after every candidate has a sample; prefetch may + stop at a low-probability reach prefix, while output-column prefetch may start early only after a + learned high survival ratio. Static conjunct schedules and position maps belong to the scanner + lifecycle; after eight warm-up samples, per-predicate clocks should be sampled only periodically. + Cache the batch SelectionVector's dense bitmap by generation so a wide lazy projection does not + rebuild the same O(batch) filter for every column. +- Single-column predicate rounds may keep previously read columns in their original row mappings. + Require one alignment compaction before multi-column, delete, or output boundaries, and expose + its time, bytes, and count instead of charging repeated movement invisibly to predicate time. + Reused compaction masks must be fully cleared when coordinate-space sizes shrink. +- PLAIN BYTE_ARRAY must not parse lengths in both decoder and destination column. Review the direct + payload-offset/cumulative-offset contract, uint32 overflow checks, surviving-span coverage, and + the legacy consumer fallback for non-string logical types. +- Production PageIndex planning must consume native Compact Thrift ColumnIndex/OffsetIndex objects. + Coalesce adjacent serialized index ranges and transfer validated OffsetIndexes into execution so + the Row Group does not read them twice. Arrow PageIndexReader is a test oracle; a production Arrow + metadata adapter or rebuilt `FileMetaData` tree is a review blocker. +- Fixed-width conversion fast paths may remove row branches only when the source domain provably + fits the target domain. Narrowing, timestamp, decimal scaling, strict rollback, and non-strict + NULL marking must retain corrupt-value tests. +- Recursive reader-profile publication and retained-scratch inspection should be amortized, but Row + Group reset/EOF/close must force the final profile delta before reader destruction. +- Accumulated lazy-column skips must not allocate one dense byte per rejected prefix row. Require a + fixed chunk bound (including multiple lazy columns) or a level-only/all-filtered cursor contract. - Register Parquet Page Cache ranges only for surviving projected Column Chunks, require a stable file-version key, and assess FileCache, MergeRange, prefetch, requests, and read amplification together. - Require counters for Statistics/Dictionary/Bloom pruning, Page Index selected ranges and skipped - rows/pages, raw and filtered rows, dictionary-row filtering, lazy-read savings, cache sources, and - remote I/O. + rows/pages, raw and filtered rows, dictionary-row filtering, PLAIN direct-predicate batches/rows, + lazy-read savings, cache sources, and remote I/O. - Differential tests must cover absent/invalid statistics, missing or partial Page Index, mixed dictionary/plain encoding, Bloom false positives, NULL/NaN/type conversion, cross-Page batches, nested/repeated columns, multiple Row Groups/Splits, and all/none filtered. diff --git a/docs/file-scanner-v2-design.md b/docs/file-scanner-v2-design.md index 66b96de1204d14..b2fe43ae76f6b0 100644 --- a/docs/file-scanner-v2-design.md +++ b/docs/file-scanner-v2-design.md @@ -282,6 +282,23 @@ Scan optimization remains maintainable only when costs are visible, sources are and failure semantics are explicit. V2 provides three complementary views: Query Profile, query resource context, and global metrics. +Query Profile uses one stable ownership tree: + +```text +FileScannerV2 +└── TableReader + └── FileReader + ├── format-specific reader (ParquetReader, OrcReader, ...) + └── IO +``` + +Scanner lifecycle and Split scheduling are charged to `FileScannerV2`; table-schema restoration, +delete handling, and reader lifecycle are charged to `TableReader`; metadata, index, decode, and +materialization are charged to `FileReader` and its format subtree; physical reads, bytes, cache +waits, and remote/local attribution remain under `IO`. Every executable lifecycle path needs a +timer, and cumulative child-reader statistics must be published at batch boundaries as well as +close, so an active slow query never presents an unexplained timing gap. + ```mermaid flowchart LR R[FileReader and FileCache Raw Statistics] --> P[Query Profile] diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 63f63819670dd2..70c041a89ccd48 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -25,8 +25,9 @@ predicate columns for surviving ranges, and finally defer output-column reads un - **Layered caches:** File-block cache, Parquet page cache, condition-result cache, and merged small I/O solve different problems and are not interchangeable. -**Scope:** This document focuses on the FileScannerV2 Parquet Reader design and core pipeline. It -does not cover Arrow decoder internals, complex-type reconstruction, or expression implementation. +**Scope:** This document covers the FileScannerV2 Parquet Reader pipeline, the native page/decode +kernel, selection-aware materialization, and complex-type reconstruction contracts. Expression +implementation details remain outside its scope. ## 2. Overall Architecture @@ -40,9 +41,10 @@ flowchart TB B --> C[TableReader
Schema Mapping, Partition/Default Values, Predicate Localization] C --> D[ParquetReader
Footer/Schema and Row Group Scan Planning] D --> E[ParquetScanScheduler
Row Group Lifecycle and Batch Reads] - E --> F[ParquetColumnReader
Page Skipping, Decompression, Decoding, Materialization] - F --> G[ParquetFileContext / Arrow RandomAccessFile
Page Cache, MergeRange, Prefetch] - G --> H[Doris FileReader / FileCache / Remote FS] + E --> F[ParquetColumnReader
Persistent Column State and Complex Reconstruction] + F --> G[Native ColumnChunk / Page / Encoding Decoders
Selection-aware Doris Materialization] + G --> H[ParquetFileContext / Stream Adapter
Page Cache, MergeRange, Prefetch] + H --> I[Doris FileReader / FileCache / Remote FS] ``` | Layer | Core responsibilities | Responsibilities intentionally excluded | @@ -58,6 +60,64 @@ flowchart TB > layer can use footer, page index, dictionary, and other format knowledge while upper layers retain > uniform scan semantics. +### 2.1 Native Reader Target and Migration State + +The target execution path does not use Arrow builders or `ReadRecords` to decode data pages. Doris +owns the Column Chunk, page, level, encoding, selection, conversion, and materialization state, and +writes directly into Doris columns. This follows the same broad separation used by DuckDB: metadata +planning is distinct from a persistent per-column reader, and page/encoding decoders expose narrow +cursor-based contracts rather than an Arrow array as an intermediate result. + +V1 remains the differential baseline, but v2 owns an independent reader implementation. Its page +and encoding algorithms started from proven Doris behavior and are maintained under the v2 tree; +the production v2 path never instantiates the v1 `ParquetColumnReader`: + +| Stage | State and boundary | +| --- | --- | +| Native encoding kernel | V2 owns the Column Chunk, page, level, and encoding decoders under `be/src/format_v2/parquet/reader/native/`. Decoders expose raw fixed/binary spans, validated dictionary indices, and skip operations; they never write Doris columns. | +| Logical materialization | `DataTypeSerDe` interprets Parquet physical/logical metadata and writes decoded spans directly into the final Doris column. Typed dictionaries and scratch are persistent per leaf reader. | +| Native column reader | `NativeColumnReader` is persistent for one top-level column and Row Group. It owns selection/filter/dictionary scratch and drives the native decoder directly into the final Doris column. | +| Complex reconstruction | Choose a Dremel shape-owning leaf per requested parent-row range; derive parent offsets/nulls once from it and keep sibling leaf streams aligned to the same parent rows. | +| Metadata and planning | Replace Arrow footer/schema/Row Group metadata dependencies with native Thrift-derived objects while preserving the existing planner, index, cache, and split contracts. | +| Compatibility removal | Production v2 has no Arrow data-read or metadata adapter and never falls back to the v1 reader; unsupported combinations return explicit errors. Arrow remains only in test oracles and fixture writers. | + +Data-page value scans and levels-only aggregate scans no longer use Arrow `RecordReader`, arrays, or +builders. V2 parses and owns the Thrift footer, schema parser, field-ID assignment, physical schema, +statistics, dictionary metadata, bloom filters, and page indexes. Production planning never builds +an Arrow metadata tree; Arrow is limited to fixture writers and differential test oracles. + +All new integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the correctness +and performance control. Compatibility is demonstrated through differential tests and the explicit +type/encoding matrix, not through a runtime dependency on v1 code. + +### 2.2 Native Interface Ownership + +```mermaid +flowchart LR + A[Native Footer / Physical Schema] --> B[RowGroupReadPlan] + B --> C[Persistent ParquetColumnReader] + C --> D[Persistent ColumnChunkReader] + D --> E[Page Reader + Level Decoders] + E --> F[Encoding Decoder] + F --> G[ColumnSelectVector] + G --> H[Doris MutableColumn] + C --> I[Shared Complex Level Plan] + I --> G +``` + +- **Physical schema contract:** Immutable physical type, fixed length, maximum definition and + repetition levels, and repeated-parent definition threshold. It owns no Arrow descriptor and + borrows no temporary table-schema object. +- **Persistent column state:** Page/decompress buffers, dictionary decoder, SerDe/conversion state, + null maps, selection ranges, binary-value references, and builder capacity live with the column + reader and are logically reset rather than recreated for each batch. +- **Decoder contract:** Consume a known number of logical level entries and encoded payload values, + then materialize only selected rows. Decoders do not decide table projection, predicate meaning, + or parent complex offsets. +- **Complex plan contract:** The shape-owning leaf's definition/repetition levels are parsed once + into parent-row boundaries and child-presence/null decisions. Sibling physical streams consume + the same parent-row range and validate their counts so offsets and null maps cannot drift. + ## 3. From File Open to Scan Plan After a reader receives a Split, it opens the file and builds the scan plan. This phase determines @@ -75,7 +135,7 @@ sequenceDiagram FS->>TR: prepare/open split TR->>TR: Map schema and localize predicates TR->>PR: FileScanRequest - PR->>FC: Open FileReader + PR->>FC: Open FileReader / native stream adapter FC->>META: Read footer and schema META-->>PR: Row Group / Column Chunk metadata PR->>PLAN: Candidate Row Groups and local predicates @@ -94,8 +154,17 @@ sequenceDiagram delete conjuncts, and local column-position mappings. - **RowGroupReadPlan:** Records the Row Group, its file-global starting row, `selected_ranges` produced by page-index pruning, and the `page_skip_plan` for each leaf column. -- **ParquetFileContext:** Adapts Doris FileReader to Arrow RandomAccessFile and owns Page Cache, - FileCache prefetch, and MergeRange routing. +- **ParquetFileContext:** Adapts Doris FileReader to the native metadata/data stream interface and + owns Page Cache, FileCache prefetch, and MergeRange routing. Its immutable footer cache entry owns + the V2 `NativeFieldDescriptor` tree directly; repeated V2 opens neither re-read the footer nor + serialize and parse another metadata representation. + +For every selected Row Group, native dictionary/index probes finish on the cached metadata tree first. The +scheduler then computes projected physical Column Chunk ranges and installs one shared native +`MergeRangeFileReader` when their average size is below v1's small-I/O threshold. All predicate and +lazy output readers share that wrapper; their internal `BufferedFileStreamReader` prefetch is +disabled in this mode, avoiding duplicate buffers. Large chunks and in-memory files use the base +reader, while remote FileCache prefetch remains the non-MergeRange path. > Planning intentionally proceeds from cheap to expensive. Split and metadata pruning reduce the > candidate set before finer indexes are read for surviving Row Groups, avoiding index I/O for data @@ -279,6 +348,258 @@ flowchart LR > output columns must decode and copy. This is the main benefit of lazy materialization in a > columnar format. +### 7.1 Selection and Cursor Contract + +The native decoder receives a sorted selection over logical rows. Logical positions include nulls; +they are not offsets into the encoded non-null payload. Definition levels and selection are merged +as two ordered streams into four run types: + +| Run | Consume logical level entries | Consume encoded payload | Append output | +| --- | --- | --- | --- | +| Selected non-null (`CONTENT`) | Yes | Yes | Value | +| Selected null (`NULL_DATA`) | Yes | No | Default plus null-map bit | +| Filtered non-null (`FILTERED_CONTENT`) | Yes | Yes or decoder-native skip | No | +| Filtered null (`FILTERED_NULL`) | Yes | No | No | + +The contract keeps three counts explicit: logical level entries consumed, encoded non-null values +consumed, and Doris output values appended. The page reader may cross page boundaries while +satisfying one batch, but all three counts must be aligned when it returns. A dense identity +selection, an empty selection, and an arbitrary fragmented selection use the same decoder API. +Selection inputs are borrowed only for the duration of the decode call. + +For a flat leaf, the fast path is valid only when the maximum repetition level is zero. It decodes +definition-level runs and builds the four-way selection plan in persistent scratch whose capacity +survives adaptive batch changes. The plan is normalized to sorted physical non-NULL ranges and +enters `DataTypeSerDe` and the encoding decoder once. If selected NULLs are interleaved, the reader +separately records their logical positions, decodes the compact physical payload, and expands the +final scalar column backwards in place. This is the hybrid selection path: it keeps the dense +direct-materialization path, but moves sparse range traversal inside the concrete decoder instead +of repeatedly constructing a SerDe consumer for every selected or NULL run. + +The encoding chooses the cheapest inner loop. PLAIN fixed-width values gather ranges with bulk +copies; PLAIN BYTE_ARRAY scans every length once and publishes compact source offsets, cumulative +output offsets, and coalesced surviving spans directly to the SerDe; dictionary encoding validates +IDs in bounded batches and either gathers repeated/literal runs directly or builds one compact +selected-ID batch when a sparse lookup would exceed the L2 cache; BOOLEAN, +DELTA, and BYTE_STREAM_SPLIT batch-decode or reconstruct their stateful stream and compact selected +values. This follows DuckDB's vector-at-a-time principle while preserving Doris's separate +Decoder/SerDe ownership boundary. A filtered non-null value always advances and validates encoding +state even when it is not copied, preventing the next batch from decoding shifted values. + +Scalar destinations with in-place expansion no longer use the per-run NULL fallback. The fallback +remains only for unsupported destination shapes and is visible through +`HybridSelectionNullFallbackBatches`; it is a correctness boundary, not an Arrow fallback. + +### 7.2 Page and Encoding Kernel + +The native page reader parses Page V1 and Page V2 headers, obtains level streams and payload bytes, +decompresses only the required region, and installs a decoder selected from physical type plus data +encoding. Page V1 and V2 differ in where levels are stored and which bytes are compressed; after +that parsing step both feed the same level and value-decoder contracts. + +| Encoding family | Native responsibility | +| --- | --- | +| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; identical POD types append by bulk copy, dense fixed-length strings use one byte-span copy plus offset synthesis, sparse fixed-width ranges consume contiguous spans directly, and variable strings reuse decoder-produced offsets without a `StringRef[]` staging pass | +| RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, validate IDs in bounded batches, fuse RLE runs with direct gather for cache-resident dictionaries, and use one compact selected-ID gather for large sparse dictionaries | +| RLE / BIT_PACKED levels | Decode definition/repetition levels and preserve runs across page and batch boundaries | +| DELTA_BINARY_PACKED | Preserve block/mini-block state, decode one page-fragment batch, and compact selected values in-place | +| DELTA_LENGTH_BYTE_ARRAY | Decode lengths and byte payload in lockstep, then retain selected references only | +| DELTA_BYTE_ARRAY | Reconstruct prefix/suffix values with persistent previous-value state, then retain selected references only | +| BYTE_STREAM_SPLIT | Reassemble selected primitive lane ranges directly into one compact batch without an Arrow intermediate | + +Unsupported physical-type/encoding combinations return an explicit error. They never fall back to +Arrow and never produce a plausible result through a decoder selected only by logical Doris type. +Dictionary-to-plain transitions, multiple data pages, Page V1/V2, truncated +payloads, integer overflow, and invalid lengths/IDs are part of the unit-test matrix. + +Read and skip are the same trust boundary. Every decoder validates the requested count against its +declared/remaining values before pointer advancement, allocation, multiplication, or integer +narrowing. BYTE_ARRAY dictionaries bound the entry count by the available four-byte length prefixes +before reserving storage and validate every decoded ID. DELTA_BYTE_ARRAY requires each prefix to fit +the preceding reconstructed value and checks the reconstructed and aggregate byte lengths. BOOLEAN +RLE and DELTA skip paths operate in bounded chunks and reject short streams instead of advancing a +partially consumed cursor as if the request succeeded. + +Level bit width is only a storage bound: for a schema maximum of 2, two bits can still encode the +invalid value 3. Batch, run, and single-value level cursors therefore reject every decoded value +above the schema maximum. Nested page traversal reserves only the requested parent-row frontier; +an untrusted Page V1/V2 `num_values` cannot trigger an eager page-sized allocation before the level +payload proves those entries exist. + +Compression has an exact-size contract. Snappy's encoded uncompressed length is checked against the +destination capacity before decompression; Page V1, Page V2, and dictionary decode must then produce +exactly the size declared in the page header. For the UNCOMPRESSED codec, dictionary compressed and +uncompressed sizes must be equal. These rules turn malformed size metadata into corruption instead +of a buffer overrun, silent truncation, or plausible shifted values. + +Before a stream is created, scalar and levels-only readers call one signed, file-bounded Column +Chunk range validator. Writer compatibility derived from `created_by` may add at most 100 bytes of +file-bounded PARQUET-816 padding, or override the pre-Arrow-3 Data Page V2 compressed flag. Page +iteration ignores auxiliary `INDEX_PAGE`/unknown pages without consuming a logical data-page +ordinal. OffsetIndex is published only when its rows and non-overlapping physical page ranges are +strictly increasing, its first location equals the owning `data_page_offset`, and every location +remains inside that validated Column Chunk. The metadata anchor rejects uniformly shifted indexes +that row monotonicity alone cannot detect; otherwise sequential traversal preserves correctness. + +### 7.3 Direct Materialization and Scratch Reuse + +Encoding decoders expose contiguous physical spans and advance encoded-stream cursors. The selected +`DataTypeSerDe` consumes those spans and writes directly into the Doris mutable column. Fixed-width +types append contiguous runs. PLAIN BYTE_ARRAY publishes payload offsets, cumulative destination +offsets, and coalesced survivor spans so string columns perform larger range copies without a +`StringRef[]` staging array. Other binary encodings may use persistent references that remain valid +only while the page or dictionary buffer is pinned by the persistent leaf reader. + +The direct path materializes the projected file type. Table-schema changes, including numeric +widening, decimal precision/scale changes, and string/date conversions, are represented by +`ColumnMapper` and executed by `TableReader` after file-local predicates finish. The native reader +rejects a different requested type because accepting it would move schema evolution into physical +decode and could change predicate semantics. No intermediate physical-value batch is introduced. + +`DecodedColumnView` is not the native Parquet decoder output ABI. It describes already decoded +physical values and is useful to generic format conversion code, but routing every Parquet value +through it would recreate an intermediate materialization layer. The v2 native path does not use it: +ColumnReader handles levels/selection, Decoder parses encoding streams, and DataTypeSerDe performs +physical/logical conversion while appending to the final column. + +Decimal and FIXED_LEN_BYTE_ARRAY direct paths validate the physical byte width, decode big-endian +two's-complement values with correct sign extension, and apply precision/scale conversion exactly +once. Decimal integer and binary inputs stay in a 256-bit intermediate through exact scale-down, +scale-up overflow, and target-precision checks; discarded non-zero digits are conversion failures, +and narrowing happens only after success. Date, timestamp, INT96, unsigned annotations, +CHAR/VARCHAR, and timezone conversions retain +the same semantic checks as the general conversion path. A fast path is enabled only when those +checks prove the result is equivalent. In particular, legacy converted `TIMESTAMP_MILLIS` and +`TIMESTAMP_MICROS` are UTC-adjusted, while an unannotated INT64 timestamp has distinct +local/unspecified semantics; data decode and statistics pruning share this interpretation. +Timestamp conversion validates millisecond scaling before multiplication, validates INT96 +nanos-of-day before widened Julian-day arithmetic, and rejects values outside Doris years +0001-9999. Plain, dictionary, and decoded-value inputs share these bounds. + +Direct conversion also preserves load-mode error semantics. A nullable target in non-strict mode +stores the nested default and marks the exact output row NULL for numeric, date/time, timestamp, and +decimal conversion failures. Strict mode and non-nullable targets return the error. Typed +dictionary materialization records failing dictionary entries once and propagates those failures +through decoded dictionary IDs, so dictionary and plain pages have identical behavior. + +The typed dictionary is materialized through the logical SerDe once per decoder dictionary +generation. Dictionary-entry predicate evaluation, dictionary-ID row filtering, and surviving +row-value flattening reuse that same Doris column. A Row Group, file, type, or dictionary-generation +change invalidates it, and every ID is checked before access. Cache-resident or dense reads stream +validated RLE runs directly into the destination column without a page-sized ID vector. Sparse +reads whose dictionary exceeds L2 retain only selected IDs, then perform one gather; malformed late +IDs roll back the destination to its pre-batch size. + +The persistent leaf reader owns reusable conversion objects, null map, selection ranges, +definition/repetition levels, binary references, dictionary state, decompression buffers, and Doris +column capacity. Logical sizes are reset at batch boundaries and normal-size capacity is retained. +After the top-level complex reader has consumed the level plan, retained capacity above the 4 MiB +high-water limit becomes eligible for release. The reader accounts separately for active and +retained bytes, including decoder-owned buffers: active oversized scratch is never released, and +eligible capacity is released only after three consecutive ordinary/idle batches. This hysteresis +prevents a repeated-value outlier from being pinned for the Row Group without turning a legitimate +large steady-state batch into allocate/free thrash. The native path does not create an Arrow +builder or Arrow array. + +### 7.4 Complex Types and Parent Shape Plans + +Repeated Parquet leaves cannot interpret a requested parent-row count as a leaf-value count. One +parent row may contain zero, one, or many level entries, and its final entry can reside on the next +page. The complex reader therefore builds a parent shape plan from one owning leaf with: + +- parent-row start/end boundaries derived from repetition levels; +- ancestor-null, collection-null, empty-collection, element-null, and present-value decisions from + definition thresholds; +- child payload positions and selected parent rows; +- cross-page continuation state for an unfinished parent row. + +ARRAY and MAP offsets and null maps are derived from that plan. Parquet stores separate level +streams for separate physical leaves, so sibling readers still consume their own streams; they must +advance over the same parent-row range and validate their payload counts instead of redefining the +parent shape. STRUCT uses a representative present leaf for its null map and parent count. MAP uses +the key leaf as the entry-shape owner, requires the materialized key and value columns to match that +outer entry count, and validates Parquet's non-null key requirement rather than repairing it. Raw +key/value repetition vectors are intentionally not compared because a nested MAP value owns +additional repetition levels. STRUCT siblings similarly normalize repetition to the current parent +boundary; deeper child collection repetition cannot redefine or invalidate the sibling shape. + +Level scratch grows with level entries that were actually decoded, while its initial reservation is +bounded by the requested parent-row frontier rather than the page header's untrusted value count. +Long repeated rows can still grow incrementally beyond that frontier, and long null/non-null runs +are split into representable internal runs without introducing a new row boundary. Tests cover null +ancestors, empty collections, null elements/values, nested +STRUCT-in-ARRAY and ARRAY-in-STRUCT shapes, sibling page misalignment, and rows spanning pages and +batches. + +#### Complex-reader interface and materialization cost + +The original v2 prototype placed an Arrow-decoded leaf batch and separate load/build/consume phases +between encoded pages and Doris columns. That design required implicit phase ordering, duplicated +level traversal in ARRAY/MAP/STRUCT wrappers, and retained decoded binary payload even when a caller +needed only nullability. The prototype hierarchy and its batch container have been removed. + +The production boundary is now the same compact cursor contract used for scalar columns: +`NativeColumnReader::read/select/skip`. Its persistent native reader owns page, decoder, level, +conversion, and complex-column state and materializes the complete result directly into the caller's +Doris column. No Arrow value reader, intermediate decoded leaf container, temporary nested Doris +column, or public load-before-build protocol participates in predicate or output scans. + +Internally, complex decoding still has to solve the parent-shape problem. For example, levels +representing `[["a", "b"], NULL, []]` produce entry counts `[2, 0, 0]`, parent nulls `[0, 1, 0]`, +and string payload ordinals `[0, 1]`. MAP uses the key leaf as the entry-shape owner and validates +the value leaf against it; STRUCT children advance in parent-row lockstep. This state belongs behind +the native reader boundary rather than in caller-visible phases. + +When schema evolution makes every projected STRUCT child missing, the native reader retains one +physical reference leaf solely for shape. Its levels-only interface advances definition, +repetition, and encoded payload cursors (including dictionary-index validation), counts STRUCT +instances from the parent thresholds, and discards the payload without allocating a temporary +string or nested Doris column. + +`CountColumnReader` selects one representative leaf (the key for MAP) and uses the v2 native +`LevelReader` to consume definition/repetition levels without decoding payload values. It exposes +neither decoded values nor the ordinary scan-reader API, so COUNT pushdown cannot become a value +fallback and large complex payloads never enter aggregate state. + +Doris v1 remains the behavior/performance baseline: `read_column_data()` owns physical decode and +the collection reader consumes persistent level buffers. DuckDB provides the same useful design +principle through its single `Read(input, vector)` boundary and reusable child vectors. See DuckDB's +official [LIST reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/list_column_reader.cpp), +[STRUCT reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/struct_column_reader.cpp), +and [base column reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/column_reader.cpp). + +### 7.5 Index Coordinate Domains and Composition + +Index correctness depends on keeping its coordinate systems separate: + +| Identity | Scope | Must not be confused with | +| --- | --- | --- | +| Table/local column ID | Current file request and Doris block | Physical Parquet leaf ordinal | +| Physical leaf-column ID | Footer Row Group Column Chunk array | Logical parent STRUCT/ARRAY/MAP ordinal | +| Row Group ID | File metadata | Split ordinal or batch ordinal | +| Data-page ordinal | One Column Chunk, excluding dictionary pages | PageHeader sequence including dictionary page | +| OffsetIndex row ordinal | Logical row inside a Row Group | Encoded non-null value position | +| Selection index | Logical row inside the current batch, nulls included | Dictionary ID or compact output position | +| Dictionary-entry ID | One Column Chunk dictionary | Row ordinal, global dictionary ID, or the next Row Group's dictionary | + +Row Group statistics, dictionary pruning, Bloom, ColumnIndex/OffsetIndex, page skip plans, cache +ranges, row selection, and lazy predicate/output readers are composed in that order. Each stage may +only remove candidates already expressed in the same Row Group logical-row domain. Page ordinals +from ColumnIndex and OffsetIndex are validated together before they become `selected_ranges` and a +per-leaf physical page-skip plan. + +Dictionary row filtering starts only when metadata proves every data page in the Column Chunk uses +a dictionary encoding. The predicate is evaluated against the current dictionary to produce an +entry bitmap; decoded IDs are checked against both dictionary length and bitmap length. A mixed +dictionary/plain transition falls back before consuming data. Once selected dictionary reading has +advanced a page cursor, loss of dictionary output is corruption rather than a retry through another +path with shifted state. + +Missing optional indexes or unsupported predicate/type combinations retain rows. Malformed +offsets, inconsistent page counts, out-of-range dictionary IDs, overlapping/unsorted invalid ranges, +or an impossible cursor relationship are reported as corruption. Cache hits and misses do not +change any page, level, value, or dictionary cursor. + ## 8. Supported Indexes and Their Boundaries V2 uses native Parquet metadata and encoding information. It does not construct Doris-internal @@ -315,12 +636,12 @@ flowchart TD ## 9. Cache and I/O Optimization -Parquet V2 has four complementary cache and I/O paths: cache remote file blocks, cache serialized -Parquet ranges, cache predicate results, and merge small random reads. +Parquet V2 has five complementary cache and I/O paths: cache footer/parsed metadata, cache remote +file blocks, cache serialized Parquet ranges, cache predicate results, and merge small random reads. ```mermaid flowchart TB - A[Parquet Column Reader ReadAt] --> B{Parquet Page Cache Hit?} + A[Parquet Column Reader Range Read] --> B{Parquet Page Cache Hit?} B -- "Yes" --> C[Return Cached Serialized Range Bytes] B -- "No" --> D{MergeRange Active?} D -- "Yes" --> E[MergeRangeFileReader
Merge Adjacent Small I/O] @@ -334,11 +655,46 @@ flowchart TB | Mechanism | Cached or optimized object | Lifecycle and key | Problem addressed | | --- | --- | --- | --- | +| Footer metadata cache | V2-owned immutable Thrift metadata and native physical schema tree | Stable base file identity plus a v2 type discriminator and schema-affecting mapping options | Avoid repeated footer I/O, Thrift parsing, schema construction, and unsafe cross-version cache casts | +| Small HTTP object staging | Complete object bytes for files at or below `in_memory_file_size` | Per-reader v2 wrapper; loaded once from byte zero and released with the file context | Collapse cold small-file requests and keep footer-cache-hit scans independent of server-specific near-EOF Range behavior | | FileCache | Remote file blocks | Related to filesystem/path and file version; may hit locally or through a peer | Avoid repeated object-storage access and support background prefetch | | Parquet Page Cache | Serialized bytes within registered Column Chunk ranges | Stable file key depends on path, mtime/version, and file size; disabled when mtime is unreliable | Reduce repeated page reads and support exact/subrange coverage | | Condition Cache | Condition-surviving granule bitmap | Managed by condition and file-range context | Reuse filtering results before reading columns | | MergeRangeFileReader | Not a cache; merges small ranges into larger slices | Installed temporarily for projected chunks of the current Row Group | Reduce remote small-I/O count and request overhead | +### Footer Cache Parity with V1 + +V2 reuses the common stable file identity policy, but owns its footer parser, schema lifecycle, +metadata payload type, and cache-key suffix entirely under `format_v2`. A v2 cache entry can be +shared by v2 readers but can never be cast as a v1 metadata object. Mutable Row Group, selection, +decoder, and scratch state remains per reader. Path-only keys are insufficient. Parse failures, +short files, unsupported metadata, and schema-affecting option changes cannot populate or reuse a +successful entry. No change to the v1 parser or metadata cache behavior is required. + +On a v2 cold miss, the parser deserializes the footer once into the V2-owned Thrift object and builds +the native schema tree directly from `SchemaElement` into V2's `NativeFieldDescriptor`. Row-group statistics, +dictionary and Bloom metadata planning consume the same native tree; no Arrow `FileMetaData` tree +or serialized-footer copy is retained. Production ColumnIndex/OffsetIndex planning reads and parses +Compact Thrift indexes natively; adjacent per-leaf serialized ranges are coalesced, and validated +OffsetIndex objects are transferred into Row Group execution instead of being fetched a second +time. Arrow metadata and PageIndexReader paths remain only as test oracles. + +Before the footer lookup, v2 wraps bounded HTTP Parquet objects in an in-memory reader. The wrapper +loads from byte zero on its first physical access, whether that access is a cold footer read or a +warm data-page read after a footer-cache hit. This preserves identical cold/warm behavior for HTTP +servers that advertise Range support but answer an overlong near-EOF range with HTTP 200, and keeps +the compatibility policy out of the v1 reader. + +### Page Cache Parity with V1 + +The native path uses v1's page-cache semantics as its minimum contract: the same stable file +identity, Column Chunk/page byte ranges, compressed versus decompressed entry distinction, checksum +and decompression ownership, subrange coverage, invalidation, admission, and fallback I/O rules. +Mutable decoder and dictionary state is never cached. A hit returns immutable bytes and advances the +reader exactly as the corresponding base read would; a miss performs normal I/O before optional +admission. An alternative policy is accepted only with correctness coverage plus warm/cold, +selective/dense, local/remote benchmarks showing it is no worse than v1. + ### Why Page Cache registers only surviving chunks The footer is read before Row Group planning and before Page Cache ranges are registered, so @@ -349,8 +705,10 @@ Chunks from surviving Row Groups are registered, limiting pollution and key coun - When the base reader is CachedRemoteFileReader, predicate/output ranges for the current Row Group may be prefetched into FileCache. -- When average projected chunks are small and the reader is not in-memory, install - MergeRangeFileReader so subsequent Arrow `ReadAt` calls actually use merged reads. +- After dictionary probing, small projected chunks share one Row-Group-scoped + MergeRangeFileReader on the native data-page path. Large chunks and in-memory files keep the base + reader. Native metadata/index reads use explicit immutable ranges and never own a decoder stream + cursor. - With row-level filters, prefetch predicate columns first. Prefetch non-predicate columns only after at least one row survives, avoiding unnecessary bandwidth. @@ -396,6 +754,14 @@ flowchart LR E --> F[Bound by batch_size and Selected Range] ``` +Changing the requested row cap changes only the amount of work performed by the persistent reader. +It does not recreate Column Readers, Column Chunk readers, dictionaries, SerDe/conversion objects, +builders, or scratch. The probe is charged as a normal batch and estimates completed Doris rows and +bytes after table-level materialization, matching v1's measurement point. Empty or highly filtered +probes do not permanently collapse the batch size; the scheduler retains enough evidence before +updating its estimate and respects page/range boundaries without turning each fragment into a new +reader lifecycle. + ### 10.3 Aggregate Pushdown When TableReader proves that no filter or delete semantics can change the result, COUNT / MIN / MAX @@ -408,6 +774,34 @@ Without row-level filtering, output columns may be warmed together. With filteri columns first and defer non-predicate columns until at least one row survives, aligning network bandwidth with lazy materialization. +For safe single-column conjuncts, the scheduler records an exponentially weighted cost per input +row and survival ratio. Cold batches preserve declaration order; after every candidate has a sample, +the next batch orders predicates by cost per rejected row. The same learned order limits predicate +prefetch to the prefix with a meaningful probability of being reached, and a sustained high overall +survival ratio estimate may warm output chunks early. This retains correctness because only +independently safe conjuncts move and all readers still advance over the same logical batch. + +Predicate-only `INT`, `BIGINT`, `FLOAT`, and `DOUBLE` comparisons can bypass Doris-column +materialization when every value page in the Column Chunk is fixed-width PLAIN and the expression +is an exact `=`, `!=`, `<`, `<=`, `>`, or `>=` comparison with a literal. The scheduler compiles +the conjunction into physical-value predicates, the decoder compares selected non-NULL spans in +place, and the reader remaps the compact match bytes to logical nullable rows. NULL comparisons are +false. Mixed encodings, projected predicate columns, casts, logical conversions, compound +expressions, and unsupported types make the decision before either the definition-level or value +cursor advances, so they safely retain ordinary expression evaluation. An exact direct result +leaves only a row-shaped placeholder for the hidden predicate slot and avoids both predicate-column +materialization and later predicate compaction. + +Selection state is scheduler-owned across adaptive batches. Its dense filter bitmap is cached by +selection generation and batch shape, so every lazily materialized output reader consumes the same +bitmap without rebuilding an O(batch-size) array. Logical sizes reset per batch while ordinary +capacity remains reusable. + +Fully rejected batches accumulate a logical lag for lazy columns. When a later batch survives, each +lazy reader consumes that lag in at most 65,535-row dense-filter chunks. This keeps the scheduler's +single logical skip while bounding adapter-owned bitmap capacity independently of prefix length and +lazy projection width. + ## 11. Correctness, Fallback, and Capability Boundaries V2 follows a prove-before-skip rule. Missing indexes, unsupported types, expressions that cannot be @@ -442,6 +836,15 @@ split safely, or read anomalies must never change query semantics. Troubleshoot in this order: verify planning effectiveness, row filtering, lazy materialization, and then I/O/cache health. Total ScanTime alone does not identify the cause. +The visible timer hierarchy is `FileScannerV2 -> TableReader -> FileReader -> IO`; the +format-specific `ParquetReader` subtree belongs to `FileReader`. Scanner lifecycle and Split work, +table-semantic restoration, format metadata/index/decode/materialization, and physical I/O are +charged to their owning layer. Native child-reader statistics accumulate in plain integers and are +published every 16 batches. Row Group reset, EOF, and reader close force the final delta before +destroying the reader tree, so short files retain complete attribution without paying recursive +profile publication on every tiny batch. Retained-scratch inspection uses the same amortized cadence +and a Row Group remains its hard lifetime bound. + ```mermaid flowchart TD A[Slow Scan] --> B{Many Row Groups Pruned?} @@ -465,10 +868,18 @@ flowchart TD | Page index pruning | How many indexes were checked, pages/rows were pruned, ranges selected, and pages skipped? | | Dictionary row filter | How often were predicates rewritten, dictionaries read, bitmaps built, and attempts successful or rejected? | | Predicate / raw rows | How many rows were read and rejected, and was lazy materialization worthwhile? | +| Predicate compaction | Did selection-first evaluation avoid repeated movement? Inspect `PredicateCompactionTime/Bytes/Count`; single-column rounds retain row mappings and compact at multi-column/delete/output boundaries. | +| PLAIN direct predicate | How many eligible predicate-only physical batches and input rows bypassed Doris-column materialization? Inspect `PlainPredicateDirectBatches/Rows`. | +| Avoided projected I/O | How many compressed bytes from projected physical chunks were avoided? `FilteredBytes` deliberately excludes unprojected nested children. | +| Metadata lifecycle | How much time was spent reading/parsing the native footer and schema tree, natively reading/parsing page indexes, and evaluating row-group/page-index predicates? | | Parquet Page Cache | What were hit/miss/write counts and compressed/decompressed hit shapes? | | FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | | Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | | Condition Cache | How many rows were skipped early after a cache hit? | +| Native decode | How much time is spent in page parsing, decompression, levels, encoding, selection, conversion, string/fixed-binary materialization, and scratch growth? Compare `HybridSelectionBatches`, `HybridSelectionRanges`, and `HybridSelectionNullFallbackBatches` to distinguish batched sparse decode from NULL-interleaving fallback. | +| Batch fragmentation | How does `TotalBatches` divide into adaptive probes, dense, selected, empty, page-crossing, and nested/fragmented batches? | +| Index decisions | How often were statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skips attempted, accepted, conservatively rejected, or rejected as corrupt? | +| Cache lifecycle | For footer/page/file/condition caches, what were request, hit, miss, bypass, admission/write, byte, wait, and underlying-I/O counts using v1-compatible meanings? | > Interpret pruning ratios in the context of write layout. Unsorted data produces wide min/max > ranges, so Row Group/Page pruning may be ineffective even when the reader and indexes work @@ -476,14 +887,16 @@ flowchart TD ## 13. Summary -The FileScannerV2 Parquet scan pipeline has three primary threads: +The FileScannerV2 Parquet scan pipeline has four primary threads: 1. **Semantic thread:** TableReader maps table schema and predicates into stable file-local semantics, preserving schema evolution, partition columns, and missing columns. 2. **Pruning thread:** Split → Row Group → Page → Row progressively applies Runtime Filters, Statistics, Dictionary, Bloom, Page Index, and actual-value filters. -3. **I/O thread:** Predicate-first reads, SelectionVector, lazy materialization, adaptive batches, - FileCache/Page Cache/Condition Cache, and MergeRange reduce read amplification together. +3. **Decode thread:** Persistent native page/encoding readers merge Dremel levels with Selection, + reuse scratch, reconstruct complex values from shared plans, and materialize directly to Doris. +4. **I/O thread:** Predicate-first reads, adaptive batches, Footer/File/Page/Condition caches, and + MergeRange reduce read amplification together. ```mermaid flowchart LR diff --git a/extension/dbt-doris/dev-requirements.txt b/extension/dbt-doris/dev-requirements.txt index 64aa3cb96848f8..12cde5e0b8fdf2 100644 --- a/extension/dbt-doris/dev-requirements.txt +++ b/extension/dbt-doris/dev-requirements.txt @@ -37,4 +37,4 @@ pytz tox>=3.13 twine wheel -mysql-connector-python>=8.0.0,<8.3 +mysql-connector-python>=8.0.33,<8.3 diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java deleted file mode 100644 index 70070ee2cb80bb..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java +++ /dev/null @@ -1,204 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import org.apache.doris.common.jni.vec.ColumnValue; - -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericData.Fixed; -import org.apache.avro.generic.GenericFixed; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Map.Entry; -import java.util.Objects; - -public class AvroColumnValue implements ColumnValue { - - private final Object fieldData; - private final ObjectInspector fieldInspector; - - public AvroColumnValue(ObjectInspector fieldInspector, Object fieldData) { - this.fieldInspector = fieldInspector; - this.fieldData = fieldData; - } - - private Object inspectObject() { - if (fieldData instanceof ByteBuffer) { - return ((PrimitiveObjectInspector) fieldInspector).getPrimitiveJavaObject(((ByteBuffer) fieldData).array()); - } else if (fieldData instanceof Fixed) { - return ((PrimitiveObjectInspector) fieldInspector).getPrimitiveJavaObject( - ((GenericFixed) fieldData).bytes()); - } - return ((PrimitiveObjectInspector) fieldInspector).getPrimitiveJavaObject(fieldData); - } - - @Override - public boolean canGetStringAsBytes() { - return false; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean getBoolean() { - return (boolean) inspectObject(); - } - - @Override - public byte getByte() { - return (byte) inspectObject(); - } - - @Override - public short getShort() { - return (short) inspectObject(); - } - - @Override - public int getInt() { - return (int) inspectObject(); - } - - @Override - public float getFloat() { - return (float) inspectObject(); - } - - @Override - public long getLong() { - return (long) inspectObject(); - } - - @Override - public double getDouble() { - return (double) inspectObject(); - } - - @Override - public BigInteger getBigInteger() { - return null; - } - - @Override - public BigDecimal getDecimal() { - return (BigDecimal) inspectObject(); - } - - @Override - public String getString() { - return inspectObject().toString(); - } - - @Override - public byte[] getStringAsBytes() { - throw new UnsupportedOperationException(); - } - - @Override - public LocalDate getDate() { - // avro has no date type - return null; - } - - @Override - public LocalDateTime getDateTime() { - // avro has no dateTime type - return null; - } - - @Override - public LocalDateTime getTimeStampTz() { - return null; - } - - @Override - public byte[] getBytes() { - return (byte[]) inspectObject(); - } - - @Override - public void unpackArray(List values) { - ListObjectInspector inspector = (ListObjectInspector) fieldInspector; - List items = inspector.getList(fieldData); - ObjectInspector itemInspector = inspector.getListElementObjectInspector(); - for (Object item : items) { - AvroColumnValue avroColumnValue = null; - if (item != null) { - avroColumnValue = new AvroColumnValue(itemInspector, item); - } - values.add(avroColumnValue); - } - } - - @Override - public void unpackMap(List keys, List values) { - MapObjectInspector inspector = (MapObjectInspector) fieldInspector; - ObjectInspector keyObjectInspector = inspector.getMapKeyObjectInspector(); - ObjectInspector valueObjectInspector = inspector.getMapValueObjectInspector(); - for (Entry kv : inspector.getMap(fieldData).entrySet()) { - AvroColumnValue avroKey = null; - AvroColumnValue avroValue = null; - if (kv.getKey() != null) { - avroKey = new AvroColumnValue(keyObjectInspector, kv.getKey()); - } - if (kv.getValue() != null) { - avroValue = new AvroColumnValue(valueObjectInspector, kv.getValue()); - } - keys.add(avroKey); - values.add(avroValue); - } - } - - @Override - public void unpackStruct(List structFieldIndex, List values) { - StructObjectInspector inspector = (StructObjectInspector) fieldInspector; - List fields = inspector.getAllStructFieldRefs(); - for (Integer idx : structFieldIndex) { - AvroColumnValue cv = null; - if (idx != null) { - StructField sf = fields.get(idx); - Object o; - if (fieldData instanceof GenericData.Record) { - GenericRecord record = (GenericRecord) inspector.getStructFieldData(fieldData, sf); - o = record.get(idx); - } else { - o = inspector.getStructFieldData(fieldData, sf); - } - if (Objects.nonNull(o)) { - cv = new AvroColumnValue(sf.getFieldObjectInspector(), o); - } - } - values.add(cv); - } - } -} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroFileContext.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroFileContext.java deleted file mode 100644 index 3e3264e62919cd..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroFileContext.java +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import org.apache.avro.Schema; - -import java.util.Set; - -public class AvroFileContext { - private Schema schema; - private Set requiredFields; - private Long splitStartOffset; - private Long splitSize; - - public void setSchema(Schema schema) { - this.schema = schema; - } - - public Schema getSchema() { - return schema; - } - - public void setRequiredFields(Set requiredFields) { - this.requiredFields = requiredFields; - } - - public void setSplitStartOffset(Long splitStartOffset) { - this.splitStartOffset = splitStartOffset; - } - - public void setSplitSize(Long splitSize) { - this.splitSize = splitSize; - } - - public Long getSplitStartOffset() { - return this.splitStartOffset; - } - - public Long getSplitSize() { - return this.splitSize; - } - - public Set getRequiredFields() { - return requiredFields; - } -} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java deleted file mode 100644 index 2ef0bf1d45aad4..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroJNIScanner.java +++ /dev/null @@ -1,225 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import org.apache.doris.common.jni.JniScanner; -import org.apache.doris.common.jni.vec.ColumnType; -import org.apache.doris.common.jni.vec.TableSchema; -import org.apache.doris.thrift.TFileType; - -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.mapred.AvroWrapper; -import org.apache.avro.mapred.Pair; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.JavaUtils; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; -import org.apache.hadoop.hive.serde2.Deserializer; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.io.NullWritable; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; - -public class AvroJNIScanner extends JniScanner { - - private static final Logger LOG = LogManager.getLogger(AvroJNIScanner.class); - private final TFileType fileType; - private final String uri; - private final Map requiredParams; - private final Integer fetchSize; - private final ClassLoader classLoader; - private int[] requiredColumnIds; - private String[] columnTypes; - private String[] requiredFields; - private Set requiredFieldSet; - private ColumnType[] requiredTypes; - private AvroReader avroReader; - private final boolean isGetTableSchema; - private StructObjectInspector rowInspector; - private Deserializer deserializer; - private StructField[] structFields; - private ObjectInspector[] fieldInspectors; - private String serde; - private AvroFileContext avroFileContext; - private AvroWrapper> inputPair; - private NullWritable ignore; - private Long splitStartOffset; - private Long splitSize; - private Long splitFileSize; - - /** - * Call by JNI for get table data or get table schema - * - * @param fetchSize The size of data fetched each time - * @param requiredParams required params - */ - public AvroJNIScanner(int fetchSize, Map requiredParams) { - this.classLoader = this.getClass().getClassLoader(); - this.requiredParams = requiredParams; - this.fetchSize = fetchSize; - this.isGetTableSchema = Boolean.parseBoolean(requiredParams.get(AvroProperties.IS_GET_TABLE_SCHEMA)); - this.fileType = TFileType.findByValue(Integer.parseInt(requiredParams.get(AvroProperties.FILE_TYPE))); - this.uri = requiredParams.get(AvroProperties.URI); - if (!isGetTableSchema) { - this.columnTypes = requiredParams.get(AvroProperties.COLUMNS_TYPES) - .split(AvroProperties.COLUMNS_TYPE_DELIMITER); - this.requiredFields = requiredParams.get(AvroProperties.REQUIRED_FIELDS) - .split(AvroProperties.FIELDS_DELIMITER); - this.requiredFieldSet = new HashSet<>(Arrays.asList(requiredFields)); - this.requiredTypes = new ColumnType[requiredFields.length]; - this.serde = requiredParams.get(AvroProperties.HIVE_SERDE); - this.structFields = new StructField[requiredFields.length]; - this.fieldInspectors = new ObjectInspector[requiredFields.length]; - this.inputPair = new AvroWrapper<>(null); - this.ignore = NullWritable.get(); - this.splitStartOffset = Long.parseLong(requiredParams.get(AvroProperties.SPLIT_START_OFFSET)); - this.splitSize = Long.parseLong(requiredParams.get(AvroProperties.SPLIT_SIZE)); - this.splitFileSize = Long.parseLong(requiredParams.get(AvroProperties.SPLIT_FILE_SIZE)); - } - } - - private void initFieldInspector() throws Exception { - requiredColumnIds = new int[requiredFields.length]; - for (int i = 0; i < requiredFields.length; i++) { - ColumnType columnType = ColumnType.parseType(requiredFields[i], columnTypes[i]); - requiredTypes[i] = columnType; - requiredColumnIds[i] = i; - } - - Properties properties = createProperties(); - deserializer = getDeserializer(new Configuration(), properties, this.serde); - rowInspector = (StructObjectInspector) deserializer.getObjectInspector(); - - for (int i = 0; i < requiredFields.length; i++) { - StructField field = rowInspector.getStructFieldRef(requiredFields[i]); - structFields[i] = field; - fieldInspectors[i] = field.getFieldObjectInspector(); - } - } - - public Properties createProperties() { - Properties properties = new Properties(); - properties.setProperty(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, - Arrays.stream(this.requiredColumnIds).mapToObj(String::valueOf).collect(Collectors.joining(","))); - properties.setProperty(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, String.join(",", requiredFields)); - properties.setProperty(AvroProperties.COLUMNS, String.join(",", requiredFields)); - properties.setProperty(AvroProperties.COLUMNS2TYPES, String.join(",", columnTypes)); - properties.setProperty(serdeConstants.SERIALIZATION_LIB, this.serde); - return properties; - } - - private Deserializer getDeserializer(Configuration configuration, Properties properties, String name) - throws Exception { - Class deserializerClass = Class.forName(name, true, JavaUtils.getClassLoader()) - .asSubclass(Deserializer.class); - Deserializer deserializer = deserializerClass.getConstructor().newInstance(); - deserializer.initialize(configuration, properties); - return deserializer; - } - - @Override - public void open() throws IOException { - Thread.currentThread().setContextClassLoader(classLoader); - switch (fileType) { - case FILE_HDFS: - this.avroReader = new HDFSFileReader(uri); - break; - case FILE_S3: - String accessKey = requiredParams.get(AvroProperties.S3_ACCESS_KEY); - String secretKey = requiredParams.get(AvroProperties.S3_SECRET_KEY); - String endpoint = requiredParams.get(AvroProperties.S3_ENDPOINT); - String region = requiredParams.get(AvroProperties.S3_REGION); - this.avroReader = new S3FileReader(accessKey, secretKey, endpoint, region, uri); - break; - default: - LOG.warn("Unsupported " + fileType.name() + " file type."); - throw new IOException("Unsupported " + fileType.name() + " file type."); - } - if (!isGetTableSchema) { - initDataReader(); - } - this.avroReader.open(avroFileContext, isGetTableSchema); - } - - private void initDataReader() { - try { - initAvroFileContext(); - initFieldInspector(); - initTableInfo(requiredTypes, requiredFields, fetchSize); - } catch (Exception e) { - LOG.warn("Failed to init avro scanner. ", e); - throw new RuntimeException(e); - } - } - - private void initAvroFileContext() { - avroFileContext = new AvroFileContext(); - avroFileContext.setRequiredFields(requiredFieldSet); - avroFileContext.setSplitStartOffset(splitStartOffset); - avroFileContext.setSplitSize(splitSize); - } - - @Override - public void close() throws IOException { - if (Objects.nonNull(avroReader)) { - avroReader.close(); - } - } - - @Override - protected int getNext() throws IOException { - int numRows = 0; - long startTime = System.nanoTime(); - for (; numRows < getBatchSize(); numRows++) { - if (!avroReader.hasNext(inputPair, ignore)) { - break; - } - GenericRecord rowRecord = (GenericRecord) avroReader.getNext(); - for (int i = 0; i < requiredFields.length; i++) { - Object fieldData = rowRecord.get(requiredFields[i]); - if (fieldData == null) { - appendData(i, null); - } else { - AvroColumnValue fieldValue = new AvroColumnValue(fieldInspectors[i], fieldData); - appendData(i, fieldValue); - } - } - } - appendDataTime += System.nanoTime() - startTime; - return numRows; - } - - @Override - protected TableSchema parseTableSchema() throws UnsupportedOperationException { - Schema schema = avroReader.getSchema(); - return AvroTypeUtils.parseTableSchema(schema); - } - -} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java deleted file mode 100644 index 416b7d25897f93..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroProperties.java +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -public class AvroProperties { - - protected static final String COLUMNS_TYPE_DELIMITER = "#"; - protected static final String FIELDS_DELIMITER = ","; - - protected static final String IS_GET_TABLE_SCHEMA = "is_get_table_schema"; - protected static final String COLUMNS_TYPES = "columns_types"; - protected static final String REQUIRED_FIELDS = "required_fields"; - protected static final String FILE_TYPE = "file_type"; - protected static final String URI = "uri"; - protected static final String S3_ACCESS_KEY = "s3.access_key"; - protected static final String S3_SECRET_KEY = "s3.secret_key"; - protected static final String S3_ENDPOINT = "s3.endpoint"; - protected static final String S3_REGION = "s3.region"; - protected static final String HIVE_SERDE = "hive.serde"; - protected static final String COLUMNS = "columns"; - protected static final String COLUMNS2TYPES = "columns.types"; - protected static final String FS_S3A_ACCESS_KEY = "fs.s3a.access.key"; - protected static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key"; - protected static final String FS_S3A_ENDPOINT = "fs.s3a.endpoint"; - protected static final String FS_S3A_REGION = "fs.s3a.region"; - protected static final String SPLIT_START_OFFSET = "split_start_offset"; - protected static final String SPLIT_SIZE = "split_size"; - protected static final String SPLIT_FILE_SIZE = "split_file_size"; - -} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java deleted file mode 100644 index ac8d4fd621df82..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroReader.java +++ /dev/null @@ -1,111 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Parser; -import org.apache.avro.file.DataFileStream; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.mapred.AvroJob; -import org.apache.avro.mapred.AvroRecordReader; -import org.apache.avro.mapred.AvroWrapper; -import org.apache.avro.mapred.Pair; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapred.FileSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Set; - -public abstract class AvroReader { - - private static final Logger LOG = LogManager.getLogger(AvroReader.class); - protected AvroRecordReader> dataReader; - protected DataFileStream schemaReader; - protected Path path; - protected FileSystem fileSystem; - - public abstract void open(AvroFileContext avroFileContext, boolean tableSchema) throws IOException; - - public abstract Schema getSchema(); - - public abstract boolean hasNext(AvroWrapper> inputPair, NullWritable ignore) throws IOException; - - public abstract Object getNext(); - - public abstract void close() throws IOException; - - protected void openSchemaReader() throws IOException { - InputStream inputStream = new BufferedInputStream(fileSystem.open(path)); - schemaReader = new DataFileStream<>(inputStream, new GenericDatumReader<>()); - if (LOG.isDebugEnabled()) { - LOG.debug("success open avro schema reader."); - } - } - - protected void openDataReader(AvroFileContext avroFileContext) throws IOException { - JobConf job = new JobConf(); - projectionSchema(job, avroFileContext); - FileSplit fileSplit = - new FileSplit(path, avroFileContext.getSplitStartOffset(), avroFileContext.getSplitSize(), job); - dataReader = new AvroRecordReader<>(job, fileSplit); - if (LOG.isDebugEnabled()) { - LOG.debug("success open avro data reader."); - } - } - - protected void projectionSchema(JobConf job, AvroFileContext avroFileContext) { - Schema projectionSchema; - Set filedNames = avroFileContext.getRequiredFields(); - Schema avroSchema = avroFileContext.getSchema(); - // The number of fields that need to be queried is the same as that of the avro file, - // so no projection is required. - if (filedNames.size() != avroSchema.getFields().size()) { - JsonObject schemaJson = new Gson().fromJson(avroSchema.toString(), JsonObject.class); - JsonArray schemaFields = schemaJson.get("fields").getAsJsonArray(); - JsonObject copySchemaJson = schemaJson.deepCopy(); - JsonArray copySchemaFields = copySchemaJson.get("fields").getAsJsonArray(); - for (int i = 0; i < schemaFields.size(); i++) { - JsonObject object = schemaFields.get(i).getAsJsonObject(); - String name = object.get("name").getAsString(); - if (filedNames.contains(name)) { - continue; - } - copySchemaFields.remove(schemaFields.get(i)); - } - projectionSchema = new Parser().parse(copySchemaJson.toString()); - } else { - projectionSchema = avroSchema; - } - AvroJob.setInputSchema(job, projectionSchema); - if (LOG.isDebugEnabled()) { - LOG.debug("projection avro schema is:" + projectionSchema.toString()); - } - } - -} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroTypeUtils.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroTypeUtils.java deleted file mode 100644 index 32ead116dd176b..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroTypeUtils.java +++ /dev/null @@ -1,126 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import org.apache.doris.common.jni.vec.TableSchema; -import org.apache.doris.common.jni.vec.TableSchema.SchemaColumn; -import org.apache.doris.thrift.TPrimitiveType; - -import com.google.common.base.Preconditions; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.apache.commons.compress.utils.Lists; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -public class AvroTypeUtils { - - private static final Logger LOG = LogManager.getLogger(AvroTypeUtils.class); - - protected static TableSchema parseTableSchema(Schema schema) throws UnsupportedOperationException { - List schemaFields = schema.getFields(); - List schemaColumns = new ArrayList<>(); - for (Field schemaField : schemaFields) { - Schema avroSchema = schemaField.schema(); - String columnName = schemaField.name(); - - SchemaColumn schemaColumn = new SchemaColumn(); - TPrimitiveType tPrimitiveType = typeFromAvro(avroSchema, schemaColumn); - schemaColumn.setName(columnName); - schemaColumn.setType(tPrimitiveType); - schemaColumns.add(schemaColumn); - } - return new TableSchema(schemaColumns); - } - - private static TPrimitiveType typeFromAvro(Schema avroSchema, SchemaColumn schemaColumn) - throws UnsupportedOperationException { - Schema.Type type = avroSchema.getType(); - switch (type) { - case ENUM: - case STRING: - return TPrimitiveType.STRING; - case INT: - return TPrimitiveType.INT; - case BOOLEAN: - return TPrimitiveType.BOOLEAN; - case LONG: - return TPrimitiveType.BIGINT; - case FLOAT: - return TPrimitiveType.FLOAT; - case FIXED: - case BYTES: - return TPrimitiveType.BINARY; - case DOUBLE: - return TPrimitiveType.DOUBLE; - case ARRAY: - SchemaColumn arrayChildColumn = new SchemaColumn(); - schemaColumn.addChildColumns(Collections.singletonList(arrayChildColumn)); - arrayChildColumn.setType(typeFromAvro(avroSchema.getElementType(), arrayChildColumn)); - return TPrimitiveType.ARRAY; - case MAP: - // The default type of AVRO MAP structure key is STRING - SchemaColumn keyChildColumn = new SchemaColumn(); - keyChildColumn.setType(TPrimitiveType.STRING); - SchemaColumn valueChildColumn = new SchemaColumn(); - valueChildColumn.setType(typeFromAvro(avroSchema.getValueType(), valueChildColumn)); - - schemaColumn.addChildColumns(Arrays.asList(keyChildColumn, valueChildColumn)); - return TPrimitiveType.MAP; - case RECORD: - List fields = avroSchema.getFields(); - List childSchemaColumn = Lists.newArrayList(); - for (Field field : fields) { - SchemaColumn structChildColumn = new SchemaColumn(); - structChildColumn.setName(field.name()); - structChildColumn.setType(typeFromAvro(field.schema(), structChildColumn)); - childSchemaColumn.add(structChildColumn); - } - schemaColumn.addChildColumns(childSchemaColumn); - return TPrimitiveType.STRUCT; - case UNION: - List nonNullableMembers = filterNullableUnion(avroSchema); - Preconditions.checkArgument(!nonNullableMembers.isEmpty(), - avroSchema.getName() + "Union child type not all nullAble type"); - List childSchemaColumns = Lists.newArrayList(); - for (Schema nullableMember : nonNullableMembers) { - SchemaColumn childColumn = new SchemaColumn(); - childColumn.setName(nullableMember.getName()); - childColumn.setType(typeFromAvro(nullableMember, childColumn)); - childSchemaColumns.add(childColumn); - } - schemaColumn.addChildColumns(childSchemaColumns); - return TPrimitiveType.STRUCT; - default: - throw new UnsupportedOperationException( - "avro format: " + avroSchema.getName() + type.getName() + " is not supported."); - } - } - - private static List filterNullableUnion(Schema schema) { - Preconditions.checkArgument(schema.isUnion(), "Schema must be union"); - return schema.getTypes().stream().filter(s -> !s.isNullable()).collect(Collectors.toList()); - } - -} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java deleted file mode 100644 index be9f355912a930..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/HDFSFileReader.java +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import org.apache.avro.Schema; -import org.apache.avro.mapred.AvroWrapper; -import org.apache.avro.mapred.Pair; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.NullWritable; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; - -import java.io.IOException; -import java.net.URI; -import java.util.Objects; - -public class HDFSFileReader extends AvroReader { - private static final Logger LOG = LogManager.getLogger(HDFSFileReader.class); - private final String url; - private AvroWrapper> inputPair; - - public HDFSFileReader(String url) { - this.url = url; - this.path = new Path(url); - } - - @Override - public void open(AvroFileContext avroFileContext, boolean tableSchema) throws IOException { - fileSystem = FileSystem.get(URI.create(url), new Configuration()); - openSchemaReader(); - if (!tableSchema) { - avroFileContext.setSchema(schemaReader.getSchema()); - openDataReader(avroFileContext); - } - } - - @Override - public Schema getSchema() { - return schemaReader.getSchema(); - } - - @Override - public boolean hasNext(AvroWrapper> inputPair, NullWritable ignore) throws IOException { - this.inputPair = inputPair; - return dataReader.next(this.inputPair, ignore); - } - - @Override - public Object getNext() { - return inputPair.datum(); - } - - @Override - public void close() throws IOException { - if (Objects.nonNull(schemaReader)) { - schemaReader.close(); - } - if (Objects.nonNull(dataReader)) { - dataReader.close(); - } - fileSystem.close(); - } -} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java deleted file mode 100644 index ae31eb815d11da..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import org.apache.avro.Schema; -import org.apache.avro.mapred.AvroWrapper; -import org.apache.avro.mapred.Pair; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.NullWritable; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; - -import java.io.IOException; -import java.net.URI; -import java.util.Objects; - -public class S3FileReader extends AvroReader { - - private static final Logger LOG = LogManager.getLogger(S3FileReader.class); - private final String bucketName; - private final String key; - private AvroWrapper> inputPair; - private final String endpoint; - private final String region; - private final String accessKey; - private final String secretKey; - private final String s3aUri; - - public S3FileReader(String accessKey, String secretKey, String endpoint, String region, String uri) - throws IOException { - this.endpoint = endpoint; - this.region = region; - S3Utils.parseURI(uri); - this.bucketName = S3Utils.getBucket(); - this.key = S3Utils.getKey(); - this.accessKey = accessKey; - this.secretKey = secretKey; - this.s3aUri = "s3a://" + bucketName + "/" + key; - } - - @Override - public void open(AvroFileContext avroFileContext, boolean tableSchema) throws IOException { - Configuration conf = new Configuration(); - if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)) { - conf.set(AvroProperties.FS_S3A_ACCESS_KEY, accessKey); - conf.set(AvroProperties.FS_S3A_SECRET_KEY, secretKey); - } - conf.set(AvroProperties.FS_S3A_ENDPOINT, endpoint); - conf.set(AvroProperties.FS_S3A_REGION, region); - path = new Path(s3aUri); - fileSystem = FileSystem.get(URI.create(s3aUri), conf); - openSchemaReader(); - if (!tableSchema) { - avroFileContext.setSchema(schemaReader.getSchema()); - openDataReader(avroFileContext); - } - } - - @Override - public Schema getSchema() { - return schemaReader.getSchema(); - } - - @Override - public boolean hasNext(AvroWrapper> inputPair, NullWritable ignore) throws IOException { - this.inputPair = inputPair; - return dataReader.next(this.inputPair, ignore); - } - - @Override - public Object getNext() { - return inputPair.datum(); - } - - @Override - public void close() throws IOException { - if (Objects.nonNull(schemaReader)) { - schemaReader.close(); - } - if (Objects.nonNull(dataReader)) { - dataReader.close(); - } - fileSystem.close(); - } -} diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java deleted file mode 100644 index 45845af3c0392e..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java +++ /dev/null @@ -1,102 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; - -public class S3Utils { - private static final String SCHEMA_S3 = "s3"; - private static final String SCHEMA_HTTP = "http"; - private static final String SCHEMA_HTTPS = "https"; - private static final String SCHEME_DELIM = "://"; - private static final String PATH_DELIM = "/"; - private static final String QUERY_DELIM = "\\?"; - private static final String FRAGMENT_DELIM = "#"; - private static String bucket; - private static String key; - - /** - * eg: - * s3: s3://bucket1/path/to/file.txt - * http: http://10.10.10.1:9000/bucket1/to/file.txt - * https: https://10.10.10.1:9000/bucket1/to/file.txt - *

- * schema: s3,http,https - * bucket: bucket1 - * key: path/to/file.txt - */ - public static void parseURI(String uri) throws IOException { - if (StringUtils.isEmpty(uri)) { - throw new IOException("s3 uri is empty."); - } - String[] schemeSplit = uri.split(SCHEME_DELIM); - String rest; - if (schemeSplit.length == 2) { - if (schemeSplit[0].equalsIgnoreCase(SCHEMA_S3)) { - // has scheme, eg: s3://bucket1/path/to/file.txt - rest = schemeSplit[1]; - String[] authoritySplit = rest.split(PATH_DELIM, 2); - if (authoritySplit.length < 1) { - throw new IOException("Invalid S3 URI. uri=" + uri); - } - bucket = authoritySplit[0]; - // support s3://bucket1 - key = authoritySplit.length == 1 ? "/" : authoritySplit[1]; - } else if (schemeSplit[0].equalsIgnoreCase(SCHEMA_HTTP) || schemeSplit[0].equalsIgnoreCase(SCHEMA_HTTPS)) { - // has scheme, eg: http(s)://host/bucket1/path/to/file.txt - rest = schemeSplit[1]; - String[] authoritySplit = rest.split(PATH_DELIM, 3); - if (authoritySplit.length != 3) { - throw new IOException("Invalid S3 HTTP URI: uri=" + uri); - } - // authority_split[1] is host - bucket = authoritySplit[1]; - key = authoritySplit[2]; - } else { - throw new IOException("Invalid S3 HTTP URI: uri=" + uri); - } - - } else if (schemeSplit.length == 1) { - // no scheme, eg: path/to/file.txt - bucket = ""; // unknown - key = uri; - } else { - throw new IOException("Invalid S3 URI. uri=" + uri); - } - - key = key.trim(); - if (StringUtils.isEmpty(key)) { - throw new IOException("Invalid S3 URI. uri=" + uri); - } - // Strip query and fragment if they exist - String[] querySplit = key.split(QUERY_DELIM); - String[] fragmentSplit = querySplit[0].split(FRAGMENT_DELIM); - key = fragmentSplit[0]; - } - - public static String getBucket() { - return bucket; - } - - public static String getKey() { - return key; - } - -} diff --git a/fe/be-java-extensions/avro-scanner/src/main/resources/package.xml b/fe/be-java-extensions/avro-scanner/src/main/resources/package.xml deleted file mode 100644 index 4bbb2610603363..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/main/resources/package.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - jar-with-dependencies - - jar - - false - - - / - true - true - runtime - - - **/Log4j2Plugins.dat - - - - - diff --git a/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/AvroTypeUtilsTest.java b/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/AvroTypeUtilsTest.java deleted file mode 100644 index 04f5fd217bf47d..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/AvroTypeUtilsTest.java +++ /dev/null @@ -1,105 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import org.apache.doris.common.jni.vec.TableSchema; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; - -public class AvroTypeUtilsTest { - private Schema allTypesRecordSchema; - private final ObjectMapper objectMapper = new ObjectMapper(); - private String result; - - @Before - public void setUp() { - result = "[{\"name\":\"aBoolean\",\"type\":2,\"childColumns\":null},{\"name\":\"aInt\",\"type\":5," - + "\"childColumns\":null},{\"name\":\"aLong\",\"type\":6,\"childColumns\":null},{\"name\":\"" - + "aFloat\",\"type\":7,\"childColumns\":null},{\"name\":\"aDouble\",\"type\":8,\"childColumns\"" - + ":null},{\"name\":\"aString\",\"type\":23,\"childColumns\":null},{\"name\":\"aBytes\",\"type\"" - + ":11,\"childColumns\":null},{\"name\":\"aFixed\",\"type\":11,\"childColumns\":null},{\"name\"" - + ":\"anArray\",\"type\":20,\"childColumns\":[{\"name\":null,\"type\":5,\"childColumns\":null}]}" - + ",{\"name\":\"aMap\",\"type\":21,\"childColumns\":[{\"name\":null,\"type\":23,\"childColumns\"" - + ":null},{\"name\":null,\"type\":5,\"childColumns\":null}]},{\"name\":\"anEnum\",\"type\":23" - + ",\"childColumns\":null},{\"name\":\"aRecord\",\"type\":22,\"childColumns\":[{\"name\":\"a\"," - + "\"type\":5,\"childColumns\":null},{\"name\":\"b\",\"type\":8,\"childColumns\":null},{\"name\":" - + "\"c\",\"type\":23,\"childColumns\":null}]},{\"name\":\"aUnion\",\"type\":22,\"childColumns\":" - + "[{\"name\":\"string\",\"type\":23,\"childColumns\":null}]}]\n"; - - Schema simpleEnumSchema = SchemaBuilder.enumeration("myEnumType").symbols("A", "B", "C"); - Schema simpleRecordSchema = SchemaBuilder.record("simpleRecord") - .fields() - .name("a") - .type().intType().noDefault() - .name("b") - .type().doubleType().noDefault() - .name("c") - .type().stringType().noDefault() - .endRecord(); - - allTypesRecordSchema = SchemaBuilder.builder() - .record("all") - .fields() - .name("aBoolean") - .type().booleanType().noDefault() - .name("aInt") - .type().intType().noDefault() - .name("aLong") - .type().longType().noDefault() - .name("aFloat") - .type().floatType().noDefault() - .name("aDouble") - .type().doubleType().noDefault() - .name("aString") - .type().stringType().noDefault() - .name("aBytes") - .type().bytesType().noDefault() - .name("aFixed") - .type().fixed("myFixedType").size(16).noDefault() - .name("anArray") - .type().array().items().intType().noDefault() - .name("aMap") - .type().map().values().intType().noDefault() - .name("anEnum") - .type(simpleEnumSchema).noDefault() - .name("aRecord") - .type(simpleRecordSchema).noDefault() - .name("aUnion") - .type().optional().stringType() - .endRecord(); - } - - @Test - public void testParseTableSchema() throws IOException { - TableSchema tableSchema = AvroTypeUtils.parseTableSchema(allTypesRecordSchema); - String tableSchemaTableSchema = tableSchema.getTableSchema(); - JsonNode tableSchemaTree = objectMapper.readTree(tableSchemaTableSchema); - - JsonNode resultSchemaTree = objectMapper.readTree(result); - Assert.assertEquals(resultSchemaTree, tableSchemaTree); - } - -} diff --git a/fe/be-java-extensions/hive-udf-shade/pom.xml b/fe/be-java-extensions/hive-udf-shade/pom.xml new file mode 100644 index 00000000000000..9d638667a67f7b --- /dev/null +++ b/fe/be-java-extensions/hive-udf-shade/pom.xml @@ -0,0 +1,114 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + hive-udf-shade + jar + Doris BE Java Extensions - Hive UDF Contract Shade + + Minimal shaded jar carrying ONLY the Hive UDF contract classes + (org.apache.hadoop.hive.ql.exec.UDF plus its load-time closure) that a + user Hive UDF extends. Replaces the ~122MB shaded Hive catalog fat jar + that java-udf previously bundled just to provide this handful of classes. + + + + + + org.apache.hive + hive-exec + core + ${hive.version} + true + + + * + * + + + + + + + hive-udf-shade + ${project.basedir}/target/ + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + package + + shade + + + + + false + + + + org.apache.hive:hive-exec + + + + + + org.apache.hive:hive-exec + + org/apache/hadoop/hive/ql/exec/UDF*.class + org/apache/hadoop/hive/ql/exec/DefaultUDFMethodResolver*.class + org/apache/hadoop/hive/ql/exec/Description*.class + org/apache/hadoop/hive/ql/metadata/HiveException*.class + + + + + + + + diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java index b014a42706ff29..4e04d5bfa1dd30 100644 --- a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java +++ b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java @@ -28,15 +28,11 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.StructLike; import org.apache.iceberg.io.CloseableIterator; -import org.apache.iceberg.types.Types.NestedField; -import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.util.SerializationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.TimeZone; import java.util.stream.Collectors; @@ -50,7 +46,7 @@ public class IcebergSysTableJniScanner extends JniScanner { private final ClassLoader classLoader; private final PreExecutionAuthenticator preExecutionAuthenticator; private final FileScanTask scanTask; - private final List fields; + private final int requiredFieldCount; private final String timezone; private CloseableIterator reader; @@ -60,15 +56,22 @@ public IcebergSysTableJniScanner(int batchSize, Map params) { Preconditions.checkArgument(serializedSplitParams != null && !serializedSplitParams.isEmpty(), "serialized_split should not be empty"); this.scanTask = SerializationUtil.deserializeFromBase64(serializedSplitParams); - String[] requiredFields = params.get("required_fields").split(","); - this.fields = selectSchema(scanTask.schema().asStruct(), requiredFields); + String requiredFieldsParam = params.get("required_fields"); + Preconditions.checkArgument(requiredFieldsParam != null && !requiredFieldsParam.isEmpty(), + "required_fields should not be empty"); + String[] requiredFields = requiredFieldsParam.split(","); + this.requiredFieldCount = requiredFields.length; this.timezone = params.getOrDefault("time_zone", TimeZone.getDefault().getID()); Map hadoopOptionParams = params.entrySet().stream() .filter(kv -> kv.getKey().startsWith(HADOOP_OPTION_PREFIX)) .collect(Collectors .toMap(kv1 -> kv1.getKey().substring(HADOOP_OPTION_PREFIX.length()), kv1 -> kv1.getValue())); this.preExecutionAuthenticator = PreExecutionAuthenticatorCache.getAuthenticator(hadoopOptionParams); - ColumnType[] requiredTypes = parseRequiredTypes(params.get("required_types").split("#"), requiredFields); + String requiredTypesParam = params.get("required_types"); + Preconditions.checkArgument(requiredTypesParam != null && !requiredTypesParam.isEmpty(), + "required_types should not be empty"); + String[] requiredTypeStrings = requiredTypesParam.split("#"); + ColumnType[] requiredTypes = parseRequiredTypes(requiredTypeStrings, requiredFields); initTableInfo(requiredTypes, requiredFields, batchSize); } @@ -106,9 +109,10 @@ protected int getNext() throws IOException { break; } StructLike row = reader.next(); - for (int i = 0; i < fields.size(); i++) { - SelectedField field = fields.get(i); - Object value = row.get(field.sourceIndex, field.field.type().typeId().javaClass()); + for (int i = 0; i < requiredFieldCount; i++) { + // FE keeps the fields requested by BE at the start of the Iceberg projection. + // FileScanTask.schema() is not the row schema for every DataTask implementation. + Object value = row.get(i, Object.class); ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone); appendData(i, columnValue); } @@ -129,35 +133,10 @@ public void close() throws IOException { } } - private static List selectSchema(StructType schema, String[] requiredFields) { - List schemaFields = schema.fields(); - List selectedFields = new ArrayList<>(); - for (String requiredField : requiredFields) { - NestedField field = schema.field(requiredField); - if (field == null) { - throw new IllegalArgumentException("RequiredField " + requiredField + " not found in schema"); - } - int sourceIndex = schemaFields.indexOf(field); - if (sourceIndex < 0) { - throw new IllegalArgumentException( - "RequiredField " + requiredField + " not found in source schema fields"); - } - selectedFields.add(new SelectedField(sourceIndex, field)); - } - return selectedFields; - } - - private static final class SelectedField { - private final int sourceIndex; - private final NestedField field; - - private SelectedField(int sourceIndex, NestedField field) { - this.sourceIndex = sourceIndex; - this.field = field; - } - } - private static ColumnType[] parseRequiredTypes(String[] typeStrings, String[] requiredFields) { + Preconditions.checkArgument(typeStrings.length == requiredFields.length, + "required_types size %s does not match required_fields size %s", + typeStrings.length, requiredFields.length); ColumnType[] requiredTypes = new ColumnType[typeStrings.length]; for (int i = 0; i < typeStrings.length; i++) { String type = typeStrings[i]; diff --git a/fe/be-java-extensions/java-udf/pom.xml b/fe/be-java-extensions/java-udf/pom.xml index 8b1e57490dccd1..37e920f976dd9c 100644 --- a/fe/be-java-extensions/java-udf/pom.xml +++ b/fe/be-java-extensions/java-udf/pom.xml @@ -49,7 +49,8 @@ under the License. org.apache.doris - hive-catalog-shade + hive-udf-shade + ${project.version} compile diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml index 4d832a0ceeafce..075b59cc8d6c23 100644 --- a/fe/be-java-extensions/pom.xml +++ b/fe/be-java-extensions/pom.xml @@ -24,11 +24,11 @@ under the License. iceberg-metadata-scanner hadoop-hudi-scanner java-common + hive-udf-shade java-udf jdbc-scanner paimon-scanner max-compute-connector - avro-scanner preload-extensions diff --git a/fe/fe-authentication/fe-authentication-handler/src/test/java/org/apache/doris/authentication/handler/AuthenticationPluginManagerTest.java b/fe/fe-authentication/fe-authentication-handler/src/test/java/org/apache/doris/authentication/handler/AuthenticationPluginManagerTest.java index 6cde32bb90f0f1..260afb30a9646c 100644 --- a/fe/fe-authentication/fe-authentication-handler/src/test/java/org/apache/doris/authentication/handler/AuthenticationPluginManagerTest.java +++ b/fe/fe-authentication/fe-authentication-handler/src/test/java/org/apache/doris/authentication/handler/AuthenticationPluginManagerTest.java @@ -70,7 +70,6 @@ void testPluginsAutoLoaded() { // Then Assertions.assertNotNull(pluginNames); Assertions.assertFalse(pluginNames.isEmpty(), "Should load at least built-in plugins"); - Assertions.assertTrue(pluginNames.contains("oidc"), "Should include oidc plugin"); Assertions.assertTrue(pluginNames.contains("password"), "Should include password plugin"); } @@ -147,10 +146,6 @@ void testGetFactory() { // Then Assertions.assertTrue(factory.isPresent()); Assertions.assertEquals("password", factory.get().name()); - - Optional oidcFactory = pluginManager.getFactory("oidc"); - Assertions.assertTrue(oidcFactory.isPresent()); - Assertions.assertEquals("oidc", oidcFactory.get().name()); } @Test diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java index aaf76f5c5da864..7cac33db8089bf 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Column.java @@ -142,6 +142,10 @@ public static Column generateBeforeValueColumn(Column column) { private boolean isKey; @SerializedName(value = "isAllowNull") private boolean isAllowNull; + // Runtime-only schema change intent; do not persist it as part of the table schema. + private transient boolean nullableSpecified; + // Runtime-only schema change intent; do not persist it as part of the table schema. + private transient boolean commentSpecified; @SerializedName(value = "isAutoInc") private boolean isAutoInc; @@ -368,6 +372,8 @@ public Column(Column column) { this.isKey = column.isKey(); this.isCompoundKey = column.isCompoundKey(); this.isAllowNull = column.isAllowNull(); + this.nullableSpecified = column.isNullableSpecified(); + this.commentSpecified = column.isCommentSpecified(); this.isAutoInc = column.isAutoInc(); this.defaultValue = column.getDefaultValue(); this.realDefaultValue = column.realDefaultValue; @@ -583,6 +589,14 @@ public boolean isAllowNull() { return isAllowNull; } + public boolean isNullableSpecified() { + return nullableSpecified; + } + + public boolean isCommentSpecified() { + return commentSpecified; + } + public boolean isAutoInc() { return isAutoInc; } @@ -595,6 +609,14 @@ public void setIsAllowNull(boolean isAllowNull) { this.isAllowNull = isAllowNull; } + public void setNullableSpecified(boolean nullableSpecified) { + this.nullableSpecified = nullableSpecified; + } + + public void setCommentSpecified(boolean commentSpecified) { + this.commentSpecified = commentSpecified; + } + public String getDefaultValue() { return this.defaultValue; } diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java index 015fecfd28a263..483f22e2425f16 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java @@ -322,11 +322,10 @@ private static void checkSupportSchemaChangeForComplexType(Type checkType, Type existingNames.add(originalField.getName()); } - // check new field name is not conflict with old field name + // check appended field names do not conflict with existing or earlier appended fields for (int i = originalFields.size(); i < otherStructType.getFields().size(); i++) { - // to check new field name is not conflict with old field name String newFieldName = otherStructType.getFields().get(i).getName(); - if (existingNames.contains(newFieldName)) { + if (!existingNames.add(newFieldName)) { throw new DdlException("Added struct field '" + newFieldName + "' conflicts with existing field"); } } diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/info/ColumnPosition.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/info/ColumnPosition.java index 7c30c725d6c3a4..5b6b53918ee59c 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/info/ColumnPosition.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/info/ColumnPosition.java @@ -18,6 +18,7 @@ package org.apache.doris.catalog.info; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.SqlUtils; import com.google.common.base.Strings; @@ -57,7 +58,7 @@ public String toSql() { if (this == FIRST) { sb.append("FIRST"); } else { - sb.append("AFTER `").append(lastCol).append("`"); + sb.append("AFTER ").append(SqlUtils.getIdentSql(lastCol)); } return sb.toString(); } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 00526509177152..655af570cfbb1c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -135,6 +135,13 @@ public class Config extends ConfigBase { description = {"Whether to check for table lock leaks"}) public static boolean check_table_lock_leaky = false; + @ConfField(mutable = false, description = {"当前 FE 节点所属的 Resource Group。可通过命令行参数 " + + "`--local_resource_group` 或环境变量 `DORIS_LOCAL_RESOURCE_GROUP` 覆盖。空字符串表示未设置。", + "The Resource Group that the current FE node belongs to. It can be overridden by the " + + "`--local_resource_group` command line option or the " + + "`DORIS_LOCAL_RESOURCE_GROUP` environment variable. An empty string means unset."}) + public static String local_resource_group = ""; + @ConfField(mutable = true, masterOnly = false, description = {"PreparedStatement stmtId starting position, used for testing only"}) public static long prepared_stmt_start_id = -1; @@ -484,7 +491,7 @@ public class Config extends ConfigBase { + "starts for the first time. You can also specify one."}) public static int cluster_id = -1; - @ConfField(description = {"Cluster token used for internal authentication."}) + @ConfField(sensitive = true, description = {"Cluster token used for internal authentication."}) public static String auth_token = ""; @ConfField(mutable = true, masterOnly = true, @@ -818,6 +825,18 @@ public class Config extends ConfigBase { // check token when download image file. @ConfField public static boolean enable_token_check = true; + @ConfField(sensitive = true, description = {"Cluster token for FE meta-service internal HTTP authentication. " + + "When set (non-empty), FE meta-service endpoints (such as image/role/check/put/journal_id) " + + "additionally require the caller to present a matching token header, on top of the existing " + + "node-host check. Empty (default) keeps the legacy behavior of node-host check only, so " + + "existing clusters and rolling upgrades are unaffected. Must be identical on all FEs and " + + "provisioned in fe.conf before enabling, otherwise FEs will reject each other.", + "FE meta-service 内部 HTTP 鉴权使用的集群 token。设置(非空)后,meta-service 端点(如 " + + "image/role/check/put/journal_id)在原有 node-host 校验之上,额外要求调用方携带匹配的 token 头。" + + "为空(默认)时维持仅 node-host 校验的旧行为,存量集群与滚动升级不受影响。必须在所有 FE 上取值一致," + + "并在启用前写入 fe.conf,否则 FE 之间会互相拒绝。"}) + public static String fe_meta_auth_token = ""; + /** * Set to true if you deploy Palo using thirdparty deploy manager * Valid options are: @@ -3071,7 +3090,7 @@ public static int metaServiceRpcRetryTimes() { public static int drop_rpc_retry_num = 200; @ConfField - public static int default_get_version_from_ms_timeout_second = 3; + public static int default_get_version_from_ms_timeout_second = 30; @ConfField(mutable = true) public static boolean enable_cloud_multi_replica = false; @@ -3470,6 +3489,38 @@ public static int metaServiceRpcRetryTimes() { "In cloud mode, the retry count when the FE request to meta service times out. Default is 1."}) public static int meta_service_rpc_timeout_retry_times = 1; + @ConfField(mutable = true, description = { + "Whether to enable QPS rate limit for RPC requests to meta service."}) + public static boolean meta_service_rpc_rate_limit_enabled = false; + + @ConfField(mutable = true, description = { + "Default QPS limit for each method (requests per second) in each cpu core, " + + "non-positive value (<= 0) means no limit"}) + public static int meta_service_rpc_rate_limit_default_qps_per_core = 50; + + @ConfField(mutable = true, + callback = MetaServiceRpcRateLimitConfigValidator.QpsConfigHandler.class, + description = { + "QPS limit config per rpc method to meta service in per cpu core, " + + "format: method1:qps1;method2:qps2, " + + "e.g.: getPartitionVersion:100;getTableVersion:100;getTabletStats:50, " + + "non-positive value (<= 0) means no limit"}) + public static String meta_service_rpc_rate_limit_qps_per_core_config + = "getPartitionVersion:500;getTableVersion:500;getTabletStats:50;beginTxn:50"; + + @ConfField(mutable = true, + callback = MetaServiceRpcRateLimitConfigValidator.PositiveIntConfigHandler.class, + description = { + "Burst window for meta service RPC rate limit in seconds. " + + "The long-term average QPS is unchanged, while calls can burst within this window."}) + public static int meta_service_rpc_rate_limit_burst_seconds = 2; + + @ConfField(mutable = true, callback = MetaServiceRpcRateLimitConfigValidator.NonNegativeLongConfigHandler.class, + description = { + "Max wait time in milliseconds when meta service RPC is rate limited, " + + "zero means fail fast."}) + public static long meta_service_rpc_rate_limit_wait_timeout_ms = 1000; + @ConfField(mutable = true, description = { "In cloud mode, the auto start and stop ignores the databases used by internal jobs, " + "such as those used for statistics. " @@ -3716,4 +3767,5 @@ public static int metaServiceRpcRetryTimes() { + "（持有主副本的桶），并在单个 tablet 写入量超过阈值（默认 200 MB）后在本地桶之间轮转。" + "可降低导入内存压力并提升随机分桶表的吞吐量，覆盖所有导入类型。"}) public static boolean enable_adaptive_random_bucket_load = true; + } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java index 192fdea7c8b109..e58fea913d7b48 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java @@ -52,6 +52,10 @@ public class ConfigBase { boolean masterOnly() default false; + // If true, the value is a secret (e.g. a token or password) and is masked in every + // config dump API (Config.dump / getConfigInfo), so it is never returned in plaintext. + boolean sensitive() default false; + String comment() default ""; VariableAnnotation varType() default VariableAnnotation.NONE; @@ -191,13 +195,26 @@ private void warnUnknownConfigKeys(String confFile, Properties props) { } } + // Placeholder returned instead of a sensitive config's real value in any dump API. + public static final String SENSITIVE_CONF_MASK = "********"; + + // Mask the value of a sensitive config (a non-empty secret) so it is never dumped in plaintext. + // An empty value is left as-is: it reveals nothing and keeps "unset" visible. + private static String maskIfSensitive(Field field, String value) { + ConfField anno = field.getAnnotation(ConfField.class); + if (anno != null && anno.sensitive() && !Strings.isNullOrEmpty(value)) { + return SENSITIVE_CONF_MASK; + } + return value; + } + public static HashMap dump() { HashMap map = new HashMap<>(); Field[] fields = confClass.getFields(); for (Field f : fields) { ConfField anno = f.getAnnotation(ConfField.class); if (anno != null) { - map.put(f.getName(), getConfValue(f)); + map.put(f.getName(), maskIfSensitive(f, getConfValue(f))); } } return map; @@ -441,6 +458,7 @@ public static synchronized List> getConfigInfo(PatternMatcher match if (confKey.equals("sys_log_dir") && Strings.isNullOrEmpty(value)) { value = System.getenv("DORIS_HOME") + "/log"; } + value = maskIfSensitive(f, value); config.add(value); config.add(f.getType().getSimpleName()); config.add(String.valueOf(confField.mutable())); diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/MetaServiceRpcRateLimitConfigValidator.java b/fe/fe-common/src/main/java/org/apache/doris/common/MetaServiceRpcRateLimitConfigValidator.java new file mode 100644 index 00000000000000..54ac359934c019 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/common/MetaServiceRpcRateLimitConfigValidator.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public final class MetaServiceRpcRateLimitConfigValidator { + private MetaServiceRpcRateLimitConfigValidator() { + } + + public static Map parseQpsPerCoreConfig(String config) throws ConfigException { + if (config == null || config.trim().isEmpty()) { + return Collections.emptyMap(); + } + + Map qpsPerCore = new HashMap<>(); + Set methods = new HashSet<>(); + for (String item : config.split(";")) { + String trimmedItem = item.trim(); + if (trimmedItem.isEmpty()) { + continue; + } + int separatorIndex = trimmedItem.indexOf(':'); + if (separatorIndex <= 0 || separatorIndex != trimmedItem.lastIndexOf(':') + || separatorIndex == trimmedItem.length() - 1) { + throw new ConfigException("Invalid format, expected method1:qps1;method2:qps2"); + } + + String methodName = trimmedItem.substring(0, separatorIndex).trim(); + String qpsText = trimmedItem.substring(separatorIndex + 1).trim(); + if (methodName.isEmpty() || qpsText.isEmpty()) { + throw new ConfigException("Invalid format, expected method1:qps1;method2:qps2"); + } + if (!methods.add(methodName)) { + throw new ConfigException("Duplicate method: " + methodName); + } + try { + qpsPerCore.put(methodName, Integer.parseInt(qpsText)); + } catch (NumberFormatException e) { + throw new ConfigException("Invalid qps for method: " + methodName, e); + } + } + return qpsPerCore; + } + + public static void validatePositive(String fieldName, int value) throws ConfigException { + if (value <= 0) { + throw new ConfigException(fieldName + " must be positive"); + } + } + + public static void validateNonNegative(String fieldName, long value) throws ConfigException { + if (value < 0) { + throw new ConfigException(fieldName + " must be non-negative"); + } + } + + public static class QpsConfigHandler extends ConfigBase.DefaultConfHandler { + @Override + public void handle(Field field, String confVal) throws Exception { + parseQpsPerCoreConfig(confVal); + super.handle(field, confVal); + } + } + + public static class PositiveIntConfigHandler extends ConfigBase.DefaultConfHandler { + @Override + public void handle(Field field, String confVal) throws Exception { + String trimmedVal = confVal.trim(); + validatePositive(field.getName(), Integer.parseInt(trimmedVal)); + super.handle(field, trimmedVal); + } + } + + public static class NonNegativeLongConfigHandler extends ConfigBase.DefaultConfHandler { + @Override + public void handle(Field field, String confVal) throws Exception { + String trimmedVal = confVal.trim(); + validateNonNegative(field.getName(), Long.parseLong(trimmedVal)); + super.handle(field, trimmedVal); + } + } +} diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java index 0d305298270ceb..d48d7ae8ed330a 100644 --- a/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java +++ b/fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java @@ -23,6 +23,8 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; public class ConfigTest { @BeforeClass @@ -34,6 +36,62 @@ public static void setUp() throws Exception { config.init(tempFile.toAbsolutePath().toString()); } + // A sensitive config (fe_meta_auth_token) must never be dumped in plaintext by any config + // API: both Config.dump() and ConfigBase.getConfigInfo() return the mask instead of the value. + @Test + public void testSensitiveConfigIsMaskedWhenSet() { + String old = Config.fe_meta_auth_token; + try { + Config.fe_meta_auth_token = "super-secret-token"; + + Map dumped = ConfigBase.dump(); + Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK, dumped.get("fe_meta_auth_token")); + + String value = configInfoValue("fe_meta_auth_token"); + Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK, value); + } finally { + Config.fe_meta_auth_token = old; + } + } + + // The legacy cluster secret auth_token is also marked sensitive, so it is masked by every + // config dump API too (it leaks through /rest/v1/config/fe otherwise). + @Test + public void testAuthTokenIsMaskedWhenSet() { + String old = Config.auth_token; + try { + Config.auth_token = "super-secret-auth-token"; + + Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK, ConfigBase.dump().get("auth_token")); + Assert.assertEquals(ConfigBase.SENSITIVE_CONF_MASK, configInfoValue("auth_token")); + } finally { + Config.auth_token = old; + } + } + + // An empty sensitive config is left as-is (no secret to hide), so "unset" stays visible. + @Test + public void testEmptySensitiveConfigIsNotMasked() { + String old = Config.fe_meta_auth_token; + try { + Config.fe_meta_auth_token = ""; + + Assert.assertEquals("", ConfigBase.dump().get("fe_meta_auth_token")); + Assert.assertEquals("", configInfoValue("fe_meta_auth_token")); + } finally { + Config.fe_meta_auth_token = old; + } + } + + private static String configInfoValue(String key) { + for (List row : ConfigBase.getConfigInfo(null)) { + if (row.get(0).equals(key)) { + return row.get(1); + } + } + throw new IllegalStateException("config not found: " + key); + } + @Test public void testSetEmptyArray() throws ConfigException { ConfigBase.setMutableConfig("s3_load_endpoint_white_list", "a,b,c"); diff --git a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClient.java b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClient.java index b945bcb6c1c075..c4cea29b4aa1c4 100644 --- a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClient.java +++ b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClient.java @@ -133,7 +133,8 @@ protected String getCatalogName(Connection conn) throws SQLException { @Override protected String[] getTableTypes() { - return new String[] {"TABLE", "VIEW", "SYSTEM TABLE"}; + // ClickHouse JDBC V2 filters engines by these vendor-specific table type names. + return new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE", "MATERIALIZED VIEW"}; } @Override @@ -145,25 +146,32 @@ protected Set getFilterInternalDatabases() { private boolean isNewClickHouseDriver(Connection conn) throws SQLException { if (isNewDriver == null) { - String driverVersion = conn.getMetaData().getDriverVersion(); - isNewDriver = driverVersion != null && !driverVersion.startsWith("0.3") - && !driverVersion.startsWith("0.4"); - if (!isNewDriver) { - // Old driver uses schema mode (not catalog), matching old JdbcClickHouseClient - databaseTermIsCatalog = false; - } else { - // New driver checks the JDBC URL for databaseterm parameter - databaseTermIsCatalog = "catalog".equalsIgnoreCase(getDatabaseTermFromUrl()); - } + DatabaseMetaData meta = conn.getMetaData(); + String driverVersion = meta.getDriverVersion(); + isNewDriver = isNewClickHouseDriverVersion(driverVersion); + databaseTermIsCatalog = isDatabaseTermCatalog( + driverVersion, meta.supportsCatalogsInDataManipulation()); } return isNewDriver; } - private String getDatabaseTermFromUrl() { - if (jdbcUrl != null && jdbcUrl.toLowerCase().contains("databaseterm=schema")) { - return "schema"; + static boolean isNewClickHouseDriverVersion(String driverVersion) { + // Custom jars may omit or rewrite manifest versions; only a known legacy version overrides metadata. + if (driverVersion == null || driverVersion.isEmpty()) { + return true; } - return "catalog"; + try { + String[] versionParts = driverVersion.split("\\."); + int majorVersion = Integer.parseInt(versionParts[0]); + int minorVersion = Integer.parseInt(versionParts[1]); + return majorVersion > 0 || (majorVersion == 0 && minorVersion >= 5); + } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) { + return true; + } + } + + static boolean isDatabaseTermCatalog(String driverVersion, boolean supportsCatalogs) { + return isNewClickHouseDriverVersion(driverVersion) && supportsCatalogs; } @Override diff --git a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClientTest.java b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClientTest.java new file mode 100644 index 00000000000000..06124b24ba0d2b --- /dev/null +++ b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClientTest.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.jdbc.client; + +import org.apache.doris.connector.jdbc.JdbcDbType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +public class JdbcClickHouseConnectorClientTest { + + private JdbcClickHouseConnectorClient createClient() { + return new JdbcClickHouseConnectorClient( + "test_catalog", + JdbcDbType.CLICKHOUSE, + "jdbc:clickhouse://localhost:8123/default", + false, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false); + } + + @Test + void testDriverVersionDetection() { + Assertions.assertTrue(JdbcClickHouseConnectorClient.isNewClickHouseDriverVersion("0.9.8")); + Assertions.assertTrue(JdbcClickHouseConnectorClient.isNewClickHouseDriverVersion("0.7.1")); + Assertions.assertFalse(JdbcClickHouseConnectorClient.isNewClickHouseDriverVersion("0.4.2")); + } + + @Test + void testDatabaseTermFollowsDriverMetadata() { + Assertions.assertFalse(JdbcClickHouseConnectorClient.isDatabaseTermCatalog("0.9.8", false)); + Assertions.assertTrue(JdbcClickHouseConnectorClient.isDatabaseTermCatalog("0.7.1", true)); + Assertions.assertFalse(JdbcClickHouseConnectorClient.isDatabaseTermCatalog("0.4.2", true)); + } + + @Test + void testUnknownDriverVersionFollowsDriverMetadata() { + for (String driverVersion : new String[] {null, "", "custom-build"}) { + Assertions.assertTrue(JdbcClickHouseConnectorClient.isNewClickHouseDriverVersion(driverVersion)); + Assertions.assertTrue(JdbcClickHouseConnectorClient.isDatabaseTermCatalog(driverVersion, true)); + Assertions.assertFalse(JdbcClickHouseConnectorClient.isDatabaseTermCatalog(driverVersion, false)); + } + } + + @Test + void testClickHouseSpecificTableTypesAreVisible() { + Assertions.assertArrayEquals( + new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE", "MATERIALIZED VIEW"}, + createClient().getTableTypes()); + } +} diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index ec53a24a75e23f..d7a7acb838707a 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -228,6 +228,10 @@ under the License. guava-testlib test + + io.github.resilience4j + resilience4j-ratelimiter + com.googlecode.java-ipv6 diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index bda379dad2f385..d313467b127287 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -42,6 +42,7 @@ import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QeService; import org.apache.doris.qe.SimpleScheduler; +import org.apache.doris.resource.Tag; import org.apache.doris.service.ExecuteEnv; import org.apache.doris.service.FrontendOptions; import org.apache.doris.tls.server.FeServerStarterFactory; @@ -89,6 +90,8 @@ public class DorisFE { private static String LOCK_FILE_PATH; private static final String LOCK_FILE_NAME = "process.lock"; + private static final String LOCAL_RESOURCE_GROUP_ENV = "DORIS_LOCAL_RESOURCE_GROUP"; + private static String effectiveLocalResourceGroupSource = "DEFAULT"; private static FileChannel processLockFileChannel; private static FileLock processFileLock; @@ -136,7 +139,8 @@ public static void start(String dorisHomeDir, String pidDir, String[] args, Star return; } - CommandLineOptions cmdLineOpts = parseArgs(args); + CommandLine commandLine = parseArgs(args); + CommandLineOptions cmdLineOpts = buildCommandLineOptions(commandLine); try { // init config @@ -146,6 +150,9 @@ public static void start(String dorisHomeDir, String pidDir, String[] args, Star // Because the path of custom config file is defined in fe.conf config.initCustom(Config.custom_config_dir + "/fe_custom.conf"); + // The command line value overrides fe.conf, so this can only run once Config is loaded. + resolveLocalResourceGroup(commandLine); + LdapConfig ldapConfig = new LdapConfig(); if (new File(dorisHomeDir + "/conf/ldap.conf").exists()) { ldapConfig.init(dorisHomeDir + "/conf/ldap.conf"); @@ -164,6 +171,8 @@ public static void start(String dorisHomeDir, String pidDir, String[] args, Star Log4jConfig.foreground = true; } Log4jConfig.initLogging(dorisHomeDir + "/conf/"); + LOG.info("effective local_resource_group={}, source={}", + Config.local_resource_group, effectiveLocalResourceGroupSource); // Add shutdown hook for graceful exit Runtime.getRuntime().addShutdownHook(new Thread(() -> { LOG.info("Received shutdown signal, starting graceful shutdown..."); @@ -361,7 +370,7 @@ private static void checkAllPorts() throws IOException { * Specify the meta version to decode log value * */ - private static CommandLineOptions parseArgs(String[] args) { + private static CommandLine parseArgs(String[] args) { CommandLineParser commandLineParser = new DefaultParser(); Options options = new Options(); options.addOption("v", "version", false, "Print the version of Doris Frontend"); @@ -382,6 +391,8 @@ private static CommandLineOptions parseArgs(String[] args) { options.addOption("c", "cluster_snapshot", true, "Specify the cluster snapshot json file"); options.addOption(Option.builder().longOpt(FeConstants.DROP_BACKENDS_KEY) .desc("When this FE becomes MASTER, drop all backends from cluster metadata (destructive)").build()); + options.addOption(null, "local_resource_group", true, + "Specify the local resource group for the current FE"); CommandLine cmd = null; try { @@ -392,6 +403,12 @@ private static CommandLineOptions parseArgs(String[] args) { System.exit(-1); } + return cmd; + } + + // Keeps the original decision order: the version / helper / image modes return before the + // recovery and drop-backends system properties are applied, so those flags stay ignored there. + private static CommandLineOptions buildCommandLineOptions(CommandLine cmd) { // version if (cmd.hasOption('v') || cmd.hasOption("version")) { return new CommandLineOptions(true, "", null, ""); @@ -494,6 +511,34 @@ private static CommandLineOptions parseArgs(String[] args) { return new CommandLineOptions(false, null, null, ""); } + // Precedence: command line option, environment variable, fe.conf. + private static void resolveLocalResourceGroup(CommandLine cmd) { + String localResourceGroup = Strings.nullToEmpty(Config.local_resource_group); + String source = localResourceGroup.isEmpty() ? "DEFAULT" : "FE_CONF"; + if (System.getenv().containsKey(LOCAL_RESOURCE_GROUP_ENV)) { + localResourceGroup = Strings.nullToEmpty(System.getenv(LOCAL_RESOURCE_GROUP_ENV)); + source = "ENV"; + } + if (cmd.hasOption("local_resource_group")) { + localResourceGroup = Strings.nullToEmpty(cmd.getOptionValue("local_resource_group")); + source = "CMDLINE"; + } + + Config.local_resource_group = localResourceGroup; + if (!localResourceGroup.isEmpty()) { + try { + // Must be a valid tag.location value, it is matched against the backends' location tag. + Tag.create(Tag.TYPE_LOCATION, localResourceGroup); + } catch (Exception e) { + // Logging is not initialized yet at this point, so report on stderr. + System.err.println("Invalid local_resource_group: " + localResourceGroup + ", " + e.getMessage()); + System.exit(-1); + } + } + // Logging is initialized later; the effective value is logged there. + effectiveLocalResourceGroupSource = source; + } + private static void printVersion() { LogUtils.stdout("Build version: " + Version.DORIS_BUILD_VERSION); LogUtils.stdout("Build time: " + Version.DORIS_BUILD_TIME); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 9f9eb03ceec676..85b2433092ecb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -415,20 +415,26 @@ private void processAlterTableForExternalTable( table.getDbName(), table.getName(), renameTableOp.getNewTableName()); } else if (alterOp instanceof AddColumnOp) { AddColumnOp addColumn = (AddColumnOp) alterOp; - table.getCatalog().addColumn(table, addColumn.getColumn(), addColumn.getColPos()); + table.getCatalog().addColumn( + table, addColumn.getColumnPath(), addColumn.getColumn(), addColumn.getColPos()); } else if (alterOp instanceof AddColumnsOp) { AddColumnsOp addColumns = (AddColumnsOp) alterOp; table.getCatalog().addColumns(table, addColumns.getColumns()); } else if (alterOp instanceof DropColumnOp) { DropColumnOp dropColumn = (DropColumnOp) alterOp; - table.getCatalog().dropColumn(table, dropColumn.getColName()); + table.getCatalog().dropColumn(table, dropColumn.getColumnPath()); } else if (alterOp instanceof RenameColumnOp) { RenameColumnOp columnRename = (RenameColumnOp) alterOp; table.getCatalog().renameColumn( - table, columnRename.getColName(), columnRename.getNewColName()); + table, columnRename.getColumnPath(), columnRename.getNewColName()); } else if (alterOp instanceof ModifyColumnOp) { ModifyColumnOp modifyColumn = (ModifyColumnOp) alterOp; - table.getCatalog().modifyColumn(table, modifyColumn.getColumn(), modifyColumn.getColPos()); + table.getCatalog().modifyColumn( + table, modifyColumn.getColumnPath(), modifyColumn.getColumn(), modifyColumn.getColPos()); + } else if (alterOp instanceof ModifyColumnCommentOp) { + ModifyColumnCommentOp modifyColumnComment = (ModifyColumnCommentOp) alterOp; + table.getCatalog().modifyColumnComment( + table, modifyColumnComment.getColumnPath(), modifyColumnComment.getComment()); } else if (alterOp instanceof ReorderColumnsOp) { ReorderColumnsOp reorderColumns = (ReorderColumnsOp) alterOp; table.getCatalog().reorderColumns(table, reorderColumns.getColumnsByPos()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index f818fcecd129e3..233cead1bdce43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -322,9 +322,9 @@ public boolean processAddColumns(AddColumnsOp addColumnsOp, OlapTable olapTable, private void addColumnRowBinlog(List rowBinlogSchema, Column newColumn, ColumnPosition columnPos, Set newColNameSet, boolean needHistoricalValue, IntSupplier columnUniqueIdSupplier) throws DdlException { - if (!newColumn.isVisible()) { - // row binlog schema is generated from visible columns only, so schema change must not - // sync hidden system columns such as sequence/delete/version/skip-bitmap columns. + if (!newColumn.isVisible() && !newColumn.isKey()) { + // Row-binlog writes visible columns plus hidden key columns. Skip hidden non-key + // system columns such as sequence/delete/version/skip-bitmap columns. return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java new file mode 100644 index 00000000000000..0e28f38de41ef5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPath.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.common.util.SqlUtils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Represents a column path used by schema change statements. + */ +public class ColumnPath { + private final ImmutableList parts; + + private ColumnPath(List parts) { + Preconditions.checkArgument(parts != null && !parts.isEmpty(), "column path is empty"); + for (String part : parts) { + Preconditions.checkArgument(part != null && !part.isEmpty(), "column path contains empty part"); + } + this.parts = ImmutableList.copyOf(parts); + } + + public static ColumnPath of(List parts) { + return new ColumnPath(parts); + } + + public static ColumnPath of(String name) { + return new ColumnPath(ImmutableList.of(name)); + } + + public static ColumnPath fromDotName(String name) { + return new ColumnPath(Arrays.asList(name.split("\\."))); + } + + public List getParts() { + return parts; + } + + public boolean isNested() { + return parts.size() > 1; + } + + public String getTopLevelName() { + return parts.get(0); + } + + public String getLeafName() { + return parts.get(parts.size() - 1); + } + + public ColumnPath getParentPath() { + Preconditions.checkState(isNested(), "top-level column path has no parent"); + return new ColumnPath(parts.subList(0, parts.size() - 1)); + } + + public String getParentPathString() { + return getParentPath().getFullPath(); + } + + public String getFullPath() { + return String.join(".", parts); + } + + public String toSql() { + return parts.stream().map(SqlUtils::getIdentSql).collect(Collectors.joining(".")); + } + + @Override + public String toString() { + return getFullPath(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java index 9d5fd94ed3e1d1..3df252e8fed56b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java @@ -497,6 +497,7 @@ public Void visitLambdaFunctionCallExpr(LambdaFunctionCallExpr expr, TExprNode m @Override public Void visitLambdaFunctionExpr(LambdaFunctionExpr expr, TExprNode msg) { msg.setNodeType(TExprNodeType.LAMBDA_FUNCTION_EXPR); + msg.setLambdaArgumentNames(expr.getNames()); return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index 8d1c0c8a4d4792..8d1c50de89080d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -509,7 +509,10 @@ public boolean ping() { } // for s3 sdk, the headObject() method does not support list "dir", // so we check FILE_REPO_INFO instead. - String path = location + "/" + joinPrefix(PREFIX_REPO, name) + "/" + FILE_REPO_INFO; + // Use getLocation() like every other repo path assembly: concrete filesystems only + // accept their native schemes, and a legacy repo location may use a compatibility + // scheme (e.g. cos:// with s3.* properties). + String path = getLocation() + "/" + joinPrefix(PREFIX_REPO, name) + "/" + FILE_REPO_INFO; try { URI checkUri = new URI(path); org.apache.doris.filesystem.FileSystem fs = acquireSpiFs(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index af6a32e993cb5d..3b1171a376ccf8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -286,7 +286,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonArray; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonArrayIgnoreNull; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonContains; -import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonExtractNoQuotes; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonHash; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonInsert; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonKeys; @@ -864,7 +863,6 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(JsonObjectFlatten.class, "json_object_flatten"), scalar(JsonQuote.class, "json_quote"), scalar(JsonUnQuote.class, "json_unquote"), - scalar(JsonExtractNoQuotes.class, "json_extract_no_quotes"), scalar(JsonHash.class, "json_hash"), scalar(JsonHash.class, "jsonb_hash"), scalar(JsonInsert.class, "json_insert", "jsonb_insert"), @@ -881,7 +879,8 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(JsonbExtractInt.class, "jsonb_extract_int", "json_extract_int", "get_json_int"), scalar(JsonbExtractIsnull.class, "json_extract_isnull"), scalar(JsonbExtractIsnull.class, "jsonb_extract_isnull"), - scalar(JsonbExtractString.class, "jsonb_extract_string", "json_extract_string", "get_json_string"), + scalar(JsonbExtractString.class, "jsonb_extract_string", "json_extract_string", + "json_extract_no_quotes", "get_json_string"), scalar(JsonbParse.class, "jsonb_parse", "json_parse"), scalar(JsonbParseErrorToNull.class, "jsonb_parse_error_to_null", "json_parse_error_to_null"), scalar(JsonbParseErrorToValue.class, "jsonb_parse_error_to_value", "json_parse_error_to_value"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 4c5937e4e55f96..dc8cac76500b15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1242,6 +1242,7 @@ public void initialize(String[] args) throws Exception { // 3. Load image first and replay edits this.editLog = new EditLog(nodeName); loadImage(this.imageDir); // load image file + seedSelfLocalResourceGroup(); migrateConstraintsFromTables(); // migrate old table-based constraints editLog.open(); // open bdb env this.globalTransactionMgr.setEditLog(editLog); @@ -1272,6 +1273,23 @@ public void initialize(String[] args) throws Exception { StmtExecutor.initBlockSqlAstNames(); } + private static void seedSelfLocalResourceGroup() { + Env env = getCurrentEnv(); + String selfNodeName = env.getNodeName(); + if (Strings.isNullOrEmpty(selfNodeName)) { + LOG.debug("skip seeding local resource group because self node name is not initialized"); + return; + } + + Frontend selfFrontend = env.frontends.get(selfNodeName); + if (selfFrontend == null) { + LOG.debug("skip seeding local resource group because self frontend {} is not found", selfNodeName); + return; + } + + selfFrontend.setLocalResourceGroup(Config.local_resource_group); + } + // wait until FE is ready. public void waitForReady() throws InterruptedException { long counter = 0; @@ -1392,6 +1410,7 @@ protected void getClusterIdAndRole() throws IOException { isFirstTimeStartUp = true; Frontend self = new Frontend(role, nodeName, selfNode.getHost(), selfNode.getPort()); + self.setLocalResourceGroup(Config.local_resource_group); // Set self alive to true, the BDBEnvironment.getReplicationGroupAdmin() will rely on this to get // helper node, before the heartbeat thread is started. self.setIsAlive(true); @@ -1874,6 +1893,7 @@ private void transferToMaster() { MetricRepo.init(); + seedSelfLocalResourceGroup(); toMasterProgress = "finished"; canRead.set(true); isReady.set(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java index 4642ac5b96c264..a343bab0ee0b7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java @@ -254,10 +254,12 @@ public static void modifyTblReplicaCount(Database database, String tblName) { InternalCatalog.INTERNAL_CATALOG_NAME, StatisticConstants.DB_NAME, tbl.getName()); - // 1. modify table's default replica num + // 1. modify table's default replica allocation Map props = new HashMap<>(); - props.put("default." + PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, - "" + StatisticConstants.STATISTIC_INTERNAL_TABLE_REPLICA_NUM); + ReplicaAllocation replicaAllocation = new ReplicaAllocation( + (short) StatisticConstants.STATISTIC_INTERNAL_TABLE_REPLICA_NUM); + props.put("default." + PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION, + replicaAllocation.toCreateStmt()); Env.getCurrentEnv().modifyTableDefaultReplicaAllocation(database, tbl, props); // 2. modify each partition's replica num diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 0174f379c4c4c1..bcd9e76e7c434c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2433,7 +2433,10 @@ public List generateTableRowBinlogSchema() { boolean needHistoricalValue = getBinlogConfig().getNeedHistoricalValue(); List beforeColumns = new ArrayList<>(); - for (Column column : getBaseSchema(false)) { + for (Column column : getBaseSchema(true)) { + if (!column.isVisible() && !column.isKey()) { + continue; + } Preconditions.checkState(!column.getType().isVariantType(), "binlog does not support VARIANT column: " + column.getName()); Preconditions.checkState(!column.isAutoInc(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index fdcb838ef6f685..bc20f860d16576 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -706,6 +706,7 @@ public class SchemaTable extends Table { .column("CURRENT_ABORT_TASK_NUM", ScalarType.createType(PrimitiveType.INT)) .column("IS_ABNORMAL_PAUSE", ScalarType.createType(PrimitiveType.BOOLEAN)) .column("COMPUTE_GROUP", ScalarType.createStringType()) + .column("FIRST_ERROR_MSG", ScalarType.createStringType()) .build()) ) .put("load_jobs", diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 9dd61f0969f72d..5a8455726b9cff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -54,6 +54,10 @@ */ public class TableProperty implements GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(TableProperty.class); + private static final String DEFAULT_REPLICATION_NUM = + "default." + PropertyAnalyzer.PROPERTIES_REPLICATION_NUM; + private static final String DEFAULT_REPLICATION_ALLOCATION = + "default." + PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION; @SerializedName(value = "properties") private Map properties; @@ -678,10 +682,23 @@ public TableProperty buildInvertedIndexFileStorageFormat() { } public void modifyTableProperties(Map modifyProperties) { + // Compatibility note: ModifyTablePropertyOperationLog persists only properties to set, not keys removed + // here. Keep its payload unchanged for this legacy repair. During a rolling FE upgrade, alter these + // properties on a table that already contains both legacy keys only after all FEs have been upgraded; + // otherwise old and new FEs may apply different effective replica settings. + removeConflictingDefaultReplicaProperty(modifyProperties); properties.putAll(modifyProperties); removeDuplicateReplicaNumProperty(); } + private void removeConflictingDefaultReplicaProperty(Map modifyProperties) { + if (modifyProperties.containsKey(DEFAULT_REPLICATION_ALLOCATION)) { + properties.remove(DEFAULT_REPLICATION_NUM); + } else if (modifyProperties.containsKey(DEFAULT_REPLICATION_NUM)) { + properties.remove(DEFAULT_REPLICATION_ALLOCATION); + } + } + public void modifyDataSortInfoProperties(DataSortInfo dataSortInfo) { properties.put(DataSortInfo.DATA_SORT_TYPE, String.valueOf(dataSortInfo.getSortType())); properties.put(DataSortInfo.DATA_SORT_COL_NUM, String.valueOf(dataSortInfo.getColNum())); @@ -690,8 +707,8 @@ public void modifyDataSortInfoProperties(DataSortInfo dataSortInfo) { public void setReplicaAlloc(ReplicaAllocation replicaAlloc) { this.replicaAlloc = replicaAlloc; // set it to "properties" so that this info can be persisted - properties.put("default." + PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION, - replicaAlloc.toCreateStmt()); + properties.remove(DEFAULT_REPLICATION_NUM); + properties.put(DEFAULT_REPLICATION_ALLOCATION, replicaAlloc.toCreateStmt()); } public ReplicaAllocation getReplicaAllocation() { @@ -968,11 +985,8 @@ public void gsonPostProcess() throws IOException { buildColumnSeqMapping(); } - // For some historical reason, - // both "dynamic_partition.replication_num" and "dynamic_partition.replication_allocation" - // may be exist in "properties". we need remove the "dynamic_partition.replication_num", or it will always replace - // the "dynamic_partition.replication_allocation", - // result in unable to set "dynamic_partition.replication_allocation". + // Historical dynamic partition metadata may contain both replica properties. Keep the allocation form by + // removing replication_num because the analyzer checks it first. private void removeDuplicateReplicaNumProperty() { if (properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM) && properties.containsKey(DynamicPartitionProperty.REPLICATION_ALLOCATION)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/stream/BaseTableStream.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/stream/BaseTableStream.java index 83b1c6773342b2..a72111730a8c76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/stream/BaseTableStream.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/stream/BaseTableStream.java @@ -81,8 +81,8 @@ public static TBinlogScanType toThrift(StreamScanType streamScanType) { @SerializedName("sir") protected boolean showInitialRows; - @SerializedName("sti") - protected StreamTableInfo streamTableInfo; + @SerializedName("bti") + protected TableStreamBaseTableInfo baseTableInfo; @SerializedName("d") private boolean disabled; @@ -102,7 +102,7 @@ public BaseTableStream() { public BaseTableStream(long id, String streamName, List fullSchema, TableIf baseTable) { super(id, streamName, TableType.STREAM, fullSchema); - this.streamTableInfo = new StreamTableInfo(baseTable); + this.baseTableInfo = new TableStreamBaseTableInfo(baseTable); this.baseTable = baseTable; this.disabled = false; this.stale = false; @@ -114,7 +114,7 @@ public BaseTableStream(String streamName, List fullSchema, TableIf baseT public TableIf getBaseTableNullable() { if (baseTable == null) { - baseTable = streamTableInfo.getTableNullable(); + baseTable = baseTableInfo.getTableNullable(); } return baseTable; } @@ -187,7 +187,7 @@ public TableIf getBaseTableOrException(java.util.function. throws E { TableIf table = getBaseTableNullable(); if (table == null) { - throw e.apply(streamTableInfo.getTableName()); + throw e.apply(baseTableInfo.getTableName()); } return table; } @@ -198,7 +198,7 @@ public TableIf getBaseTableOrNereidsAnalysisException() throws AnalysisException } public List getBaseTableFullQualifiers() { - return streamTableInfo.getFullQualifiers(); + return baseTableInfo.getFullQualifiers(); } public abstract void unprotectedCheckStreamUpdate(AbstractTableStreamUpdate update) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/stream/StreamTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/stream/TableStreamBaseTableInfo.java similarity index 94% rename from fe/fe-core/src/main/java/org/apache/doris/catalog/stream/StreamTableInfo.java rename to fe/fe-core/src/main/java/org/apache/doris/catalog/stream/TableStreamBaseTableInfo.java index d19dc24b1366f4..3571b7e3592edd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/stream/StreamTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/stream/TableStreamBaseTableInfo.java @@ -32,8 +32,8 @@ import java.util.List; import java.util.Optional; -public class StreamTableInfo { - private static final Logger LOG = LogManager.getLogger(StreamTableInfo.class); +public class TableStreamBaseTableInfo { + private static final Logger LOG = LogManager.getLogger(TableStreamBaseTableInfo.class); // for internal table we use id as identifier otherwise use name instead @SerializedName("ci") private final long ctlId; @@ -48,7 +48,7 @@ public class StreamTableInfo { @SerializedName("cn") private final String ctlName; - public StreamTableInfo(TableIf table) { + public TableStreamBaseTableInfo(TableIf table) { java.util.Objects.requireNonNull(table, "table is null"); DatabaseIf database = table.getDatabase(); java.util.Objects.requireNonNull(database, "database is null"); @@ -123,7 +123,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - StreamTableInfo that = (StreamTableInfo) o; + TableStreamBaseTableInfo that = (TableStreamBaseTableInfo) o; if (isInternalTable()) { return Objects.equal(tableId, that.tableId) && Objects.equal( dbId, that.dbId) && Objects.equal(ctlId, that.ctlId); @@ -140,7 +140,7 @@ public int hashCode() { @Override public String toString() { - return "BaseTableInfo{" + return "TableStreamBaseTableInfo{" + "tableName='" + tableName + '\'' + ", dbName='" + dbName + '\'' + ", ctlName='" + ctlName + '\'' diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/BalanceTypeEnum.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/BalanceTypeEnum.java index d66e3126d5bb59..55dbfba1cc52ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/BalanceTypeEnum.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/BalanceTypeEnum.java @@ -64,6 +64,7 @@ public static boolean isValid(String value) { */ public static BalanceTypeEnum getCloudWarmUpForRebalanceTypeEnum() { return fromString(Config.cloud_warm_up_for_rebalance_type) == null - ? ComputeGroup.DEFAULT_COMPUTE_GROUP_BALANCE_ENUM : fromString(Config.cloud_warm_up_for_rebalance_type); + ? CloudComputeGroupMeta.DEFAULT_COMPUTE_GROUP_BALANCE_ENUM + : fromString(Config.cloud_warm_up_for_rebalance_type); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/ComputeGroup.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudComputeGroupMeta.java similarity index 89% rename from fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/ComputeGroup.java rename to fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudComputeGroupMeta.java index c895f13f7a03e8..4d258ed262e391 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/ComputeGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudComputeGroupMeta.java @@ -34,8 +34,26 @@ import java.util.List; import java.util.Map; -public class ComputeGroup { - private static final Logger LOG = LogManager.getLogger(ComputeGroup.class); +/** + * FE-side in-memory metadata for a cloud compute group. + * + *

This class models both physical and virtual cloud compute groups and keeps cloud-only + * control-plane state, including the compute group type, active-standby policy, sub compute + * groups, availability timestamps, and cache warm-up properties. Its instances are refreshed + * from the meta service by {@link CloudInstanceStatusChecker} and + * {@link org.apache.doris.cloud.system.CloudSystemInfoService}. + * + *

Do not confuse this class with {@link org.apache.doris.resource.computegroup.ComputeGroup}. + * The resource-layer class is a runtime routing abstraction shared by cloud and non-cloud + * deployments: it selects backends and resolves the workload group namespace for a request. + * In contrast, this class is the long-lived cloud control-plane metadata consulted during + * routing, failover, and cache warm-up. + * + *

The meta service is the source of truth. This class is only an FE in-memory mirror and is + * not persisted in the FE edit log or image. + */ +public class CloudComputeGroupMeta { + private static final Logger LOG = LogManager.getLogger(CloudComputeGroupMeta.class); public static final String BALANCE_TYPE = "balance_type"; @@ -139,7 +157,7 @@ public Cloud.ClusterPolicy toPb() { @Setter private Map properties = new LinkedHashMap<>(ALL_PROPERTIES_DEFAULT_VALUE_MAP); - public ComputeGroup(String id, String name, ComputeTypeEnum type) { + public CloudComputeGroupMeta(String id, String name, ComputeTypeEnum type) { this.id = id; this.name = name; this.type = type; @@ -305,10 +323,10 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof ComputeGroup)) { + if (!(o instanceof CloudComputeGroupMeta)) { return false; } - ComputeGroup that = (ComputeGroup) o; + CloudComputeGroupMeta that = (CloudComputeGroupMeta) o; return unavailableSince == that.unavailableSince && availableSince == that.availableSince && id.equals(that.id) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java index 73805a950ceae8..0c7c7774445252 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java @@ -113,7 +113,7 @@ private void processVirtualClusters(List clusters) { private void handleComputeClusters(List computeClusters) { for (Cloud.ClusterPB computeClusterInMs : computeClusters) { - ComputeGroup computeGroupInFe = cloudSystemInfoService + CloudComputeGroupMeta computeGroupInFe = cloudSystemInfoService .getComputeGroupById(computeClusterInMs.getClusterId()); if (computeGroupInFe == null) { // cluster checker will sync it @@ -131,7 +131,7 @@ private void handleComputeClusters(List computeClusters) { * Compare properties between compute cluster in MS and compute group in FE, * update only the changed key-value pairs to avoid unnecessary updates. */ - private void updatePropertiesIfChanged(ComputeGroup computeGroupInFe, Cloud.ClusterPB computeClusterInMs) { + private void updatePropertiesIfChanged(CloudComputeGroupMeta computeGroupInFe, Cloud.ClusterPB computeClusterInMs) { Map propertiesInMs = computeClusterInMs.getPropertiesMap(); Map propertiesInFe = computeGroupInFe.getProperties(); @@ -181,7 +181,7 @@ private void categorizeClusters(List clusters, private void handleVirtualClusters(List virtualGroups, List computeClusters) { for (Cloud.ClusterPB virtualGroupInMs : virtualGroups) { - ComputeGroup virtualGroupInFe = cloudSystemInfoService + CloudComputeGroupMeta virtualGroupInFe = cloudSystemInfoService .getComputeGroupById(virtualGroupInMs.getClusterId()); if (virtualGroupInFe != null) { handleExistingVirtualComputeGroup(virtualGroupInMs, virtualGroupInFe); @@ -203,7 +203,7 @@ private void handleVirtualClusters(List virtualGroups, List jobIds) { + private void cancelCacheJobs(CloudComputeGroupMeta vcgInFe, List jobIds) { CacheHotspotManager cacheHotspotManager = ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr(); if (!jobIds.isEmpty()) { LOG.info("warmup-vcg cancel-cache-jobs vcgName={} activeComputeGroup={} standbyComputeGroup={} " @@ -224,7 +224,7 @@ private void cancelCacheJobs(ComputeGroup vcgInFe, List jobIds) { } } - private void checkNeedRebuildFileCache(ComputeGroup virtualGroupInFe, List jobIdsInMs) { + private void checkNeedRebuildFileCache(CloudComputeGroupMeta virtualGroupInFe, List jobIdsInMs) { CacheHotspotManager cacheHotspotManager = ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr(); // check jobIds in Ms valid, if been cancelled, start new jobs for (String jobId : jobIdsInMs) { @@ -272,7 +272,8 @@ private void checkNeedRebuildFileCache(ComputeGroup virtualGroupInFe, List subComputeGroups = clusterInMs.getClusterNamesList(); if (subComputeGroups.isEmpty() || virtualGroupInFe.getSubComputeGroups() == null) { LOG.warn("virtual compute err, please check it, verbose {}", virtualGroupInFe); @@ -396,7 +398,7 @@ private boolean areSubComputeGroupsValid(Cloud.ClusterPB clusterInMs, ComputeGro return true; } - private void diffAndUpdateComputeGroup(Cloud.ClusterPB cluster, ComputeGroup computeGroup) { + private void diffAndUpdateComputeGroup(Cloud.ClusterPB cluster, CloudComputeGroupMeta computeGroup) { // vcg rename logic, here cluster_id same, but cluster_name changed, so vcg renamed String clusterNameInMs = cluster.getClusterName(); String computeGroupNameInFe = computeGroup.getName(); @@ -490,10 +492,10 @@ private void handleNewVirtualComputeGroup(Cloud.ClusterPB cluster, List(subComputeGroups)); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(cluster.getClusterPolicy().getActiveClusterName()); policy.setStandbyComputeGroup(cluster.getClusterPolicy().getStandbyClusterNames(0)); policy.setFailoverFailureThreshold(cluster.getClusterPolicy().getFailoverFailureThreshold()); @@ -541,7 +543,7 @@ private void handleFailedSync(Cloud.ClusterPB cluster, String subClusterName, private void removeObsoleteVirtualGroups(List virtualClusters) { List msVirtualClusters = virtualClusters.stream().map(Cloud.ClusterPB::getClusterId) .collect(Collectors.toList()); - for (ComputeGroup computeGroup : cloudSystemInfoService.getComputeGroups(true)) { + for (CloudComputeGroupMeta computeGroup : cloudSystemInfoService.getComputeGroups(true)) { // in fe mem, but not in meta server if (!msVirtualClusters.contains(computeGroup.getId())) { LOG.info("virtual compute group {} will be removed.", computeGroup.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index ee9eb1eb568bef..6acaad24e0700a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -202,7 +202,7 @@ public int hashCode() { * Get the current balance type for a compute group, falling back to global balance type if not found */ private BalanceTypeEnum getCurrentBalanceType(String clusterId) { - ComputeGroup cg = cloudSystemInfoService.getComputeGroupById(clusterId); + CloudComputeGroupMeta cg = cloudSystemInfoService.getComputeGroupById(clusterId); if (cg == null) { LOG.debug("compute group not found, use global balance type, id {}", clusterId); return globalBalanceTypeEnum; @@ -219,7 +219,7 @@ private BalanceTypeEnum getCurrentBalanceType(String clusterId) { * Get the current task timeout for a compute group, falling back to global timeout if not found */ private int getCurrentTaskTimeout(String clusterId) { - ComputeGroup cg = cloudSystemInfoService.getComputeGroupById(clusterId); + CloudComputeGroupMeta cg = cloudSystemInfoService.getComputeGroupById(clusterId); if (cg == null) { return Config.cloud_pre_heating_time_limit_sec; } @@ -233,15 +233,15 @@ private int getCurrentTaskTimeout(String clusterId) { } private boolean isComputeGroupBalanceChanged(String clusterId) { - ComputeGroup cg = cloudSystemInfoService.getComputeGroupById(clusterId); + CloudComputeGroupMeta cg = cloudSystemInfoService.getComputeGroupById(clusterId); if (cg == null) { return false; } BalanceTypeEnum computeGroupBalanceType = cg.getBalanceType(); int computeGroupTimeout = cg.getBalanceWarmUpTaskTimeout(); - return computeGroupBalanceType != ComputeGroup.DEFAULT_COMPUTE_GROUP_BALANCE_ENUM - || computeGroupTimeout != ComputeGroup.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT; + return computeGroupBalanceType != CloudComputeGroupMeta.DEFAULT_COMPUTE_GROUP_BALANCE_ENUM + || computeGroupTimeout != CloudComputeGroupMeta.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT; } public CloudTabletRebalancer(CloudSystemInfoService cloudSystemInfoService) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index 25fcfbcd0d7505..6b81f084717afe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -19,6 +19,7 @@ import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.common.Config; +import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.metric.CloudMetrics; import org.apache.doris.metric.MetricRepo; import org.apache.doris.rpc.RpcException; @@ -39,6 +40,7 @@ public class MetaServiceProxy { private static final Logger LOG = LogManager.getLogger(MetaServiceProxy.class); + private static final MetaServiceRpcRateLimiter META_SERVICE_RPC_RATE_LIMITER = new MetaServiceRpcRateLimiter(); // use exclusive lock to make sure only one thread can add or remove client from // serviceMap. @@ -105,6 +107,7 @@ public Cloud.GetInstanceResponse getInstance(Cloud.GetInstanceRequest request) } try { + acquireRateLimit(methodName); final MetaServiceClient client = getProxy(); Cloud.GetInstanceResponse response = client.getInstance(request); if (MetricRepo.isInit && Config.isCloudMode()) { @@ -112,17 +115,63 @@ public Cloud.GetInstanceResponse getInstance(Cloud.GetInstanceRequest request) .update(System.currentTimeMillis() - startTime); } return response; + } catch (MetaServiceRateLimitException e) { + recordRpcRateLimited(methodName); + throw e; } catch (Exception e) { - if (MetricRepo.isInit && Config.isCloudMode()) { - CloudMetrics.META_SERVICE_RPC_ALL_FAILED.increase(1L); - CloudMetrics.META_SERVICE_RPC_FAILED.getOrAdd(methodName).increase(1L); - CloudMetrics.META_SERVICE_RPC_LATENCY.getOrAdd(methodName) - .update(System.currentTimeMillis() - startTime); - } + recordRpcFailed(methodName, startTime); throw new RpcException("", e.getMessage(), e); } } + private static long acquireRateLimit(String methodName) throws RpcException { + return META_SERVICE_RPC_RATE_LIMITER.acquire(methodName); + } + + private static long acquireRateLimit(String methodName, int permits) throws RpcException { + return META_SERVICE_RPC_RATE_LIMITER.acquire(methodName, permits); + } + + private static int getGetVersionRateLimitPermits(Cloud.GetVersionRequest request) { + if (!request.getBatchMode()) { + return 1; + } + int permits = request.hasIsTableVersion() && request.getIsTableVersion() + ? request.getTableIdsCount() + : request.getPartitionIdsCount(); + return Math.max(permits, 1); + } + + static void resetMetaServiceRpcRateLimitForTest() { + META_SERVICE_RPC_RATE_LIMITER.reset(); + } + + private static void recordRpcFailed(String methodName, long startTime) { + if (MetricRepo.isInit && Config.isCloudMode()) { + CloudMetrics.META_SERVICE_RPC_ALL_FAILED.increase(1L); + CloudMetrics.META_SERVICE_RPC_FAILED.getOrAdd(methodName).increase(1L); + CloudMetrics.META_SERVICE_RPC_LATENCY.getOrAdd(methodName) + .update(System.currentTimeMillis() - startTime); + } + } + + private static void recordRpcRateLimited(String methodName) { + if (MetricRepo.isInit && Config.isCloudMode()) { + CloudMetrics.META_SERVICE_RPC_ALL_RATE_LIMITED.increase(1L); + CloudMetrics.META_SERVICE_RPC_RATE_LIMITED.getOrAdd(methodName).increase(1L); + } + } + + private static void recordGetVersionRateLimitWait(long waitNs) { + if (waitNs <= 0) { + return; + } + SummaryProfile profile = SummaryProfile.getSummaryProfile(null); + if (profile != null) { + profile.addGetMetaVersionRateLimitWaitTime(waitNs); + } + } + public void removeProxy(String address) { LOG.warn("begin to remove proxy: {}", address); MetaServiceClient service; @@ -207,6 +256,7 @@ public Response executeRequest(String methodName, Function 1 && MetricRepo.isInit && Config.isCloudMode()) { CloudMetrics.META_SERVICE_RPC_ALL_RETRY.increase(1L); @@ -289,13 +339,11 @@ private Response executeWithMetrics(String methodName, Function getVisibleVersionAsync(Cloud.GetVersionR } try { + recordGetVersionRateLimitWait(acquireRateLimit(methodName, getGetVersionRateLimitPermits(request))); client = getProxy(); Future future = client.getVisibleVersionAsync(request); if (future instanceof com.google.common.util.concurrent.ListenableFuture) { @@ -331,12 +380,7 @@ public void onSuccess(Cloud.GetVersionResponse result) { @Override public void onFailure(Throwable t) { - if (MetricRepo.isInit && Config.isCloudMode()) { - CloudMetrics.META_SERVICE_RPC_ALL_FAILED.increase(1L); - CloudMetrics.META_SERVICE_RPC_FAILED.getOrAdd(methodName).increase(1L); - CloudMetrics.META_SERVICE_RPC_LATENCY.getOrAdd(methodName) - .update(System.currentTimeMillis() - startTime); - } + recordRpcFailed(methodName, startTime); if (finalClient != null) { finalClient.shutdown(true); } @@ -344,13 +388,11 @@ public void onFailure(Throwable t) { }, com.google.common.util.concurrent.MoreExecutors.directExecutor()); } return future; + } catch (MetaServiceRateLimitException e) { + recordRpcRateLimited(methodName); + throw e; } catch (Exception e) { - if (MetricRepo.isInit && Config.isCloudMode()) { - CloudMetrics.META_SERVICE_RPC_ALL_FAILED.increase(1L); - CloudMetrics.META_SERVICE_RPC_FAILED.getOrAdd(methodName).increase(1L); - CloudMetrics.META_SERVICE_RPC_LATENCY.getOrAdd(methodName) - .update(System.currentTimeMillis() - startTime); - } + recordRpcFailed(methodName, startTime); if (client != null) { client.shutdown(true); } @@ -358,13 +400,6 @@ public void onFailure(Throwable t) { } } - public Cloud.GetVersionResponse getVersion(Cloud.GetVersionRequest request) throws RpcException { - String methodName = request.hasIsTableVersion() && request.getIsTableVersion() ? "getTableVersion" - : "getPartitionVersion"; - return executeWithMetrics(methodName, (client) -> client.getVersion(request), - Cloud.GetVersionResponse::getStatus); - } - public Cloud.CreateTabletsResponse createTablets(Cloud.CreateTabletsRequest request) throws RpcException { return executeWithMetrics("createTablets", (client) -> client.createTablets(request), Cloud.CreateTabletsResponse::getStatus); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceRateLimitException.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceRateLimitException.java new file mode 100644 index 00000000000000..219a43d467b456 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceRateLimitException.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.rpc; + +import org.apache.doris.rpc.RpcException; + +class MetaServiceRateLimitException extends RpcException { + MetaServiceRateLimitException(String methodName, long waitTimeoutMs) { + super("", "meta service rpc rate limited, method: " + methodName + + ", wait timeout: " + waitTimeoutMs + "ms"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceRpcRateLimiter.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceRpcRateLimiter.java new file mode 100644 index 00000000000000..cfcd94076412d1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceRpcRateLimiter.java @@ -0,0 +1,278 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.rpc; + +import org.apache.doris.common.Config; +import org.apache.doris.common.ConfigException; +import org.apache.doris.common.MetaServiceRpcRateLimitConfigValidator; +import org.apache.doris.metric.CloudMetrics; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.rpc.RpcException; + +import com.google.common.collect.Maps; +import io.github.resilience4j.ratelimiter.RateLimiter; +import io.github.resilience4j.ratelimiter.RateLimiterConfig; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.Duration; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +class MetaServiceRpcRateLimiter { + private static final Logger LOG = LogManager.getLogger(MetaServiceRpcRateLimiter.class); + + private static final int CPU_CORES = Runtime.getRuntime().availableProcessors(); + private static final RateLimitConfigSnapshot EMPTY_SNAPSHOT = + new RateLimitConfigSnapshot(0, "", 1, 0); + + private final ReentrantLock configLock = new ReentrantLock(); + private final ConcurrentMap rateLimiters = Maps.newConcurrentMap(); + private volatile RateLimitConfigSnapshot currentSnapshot = EMPTY_SNAPSHOT; + private volatile Map methodQpsPerCore = Collections.emptyMap(); + + long acquire(String methodName) throws RpcException { + return acquire(methodName, 1); + } + + long acquire(String methodName, int permits) throws RpcException { + if (!Config.meta_service_rpc_rate_limit_enabled) { + return 0; + } + + RateLimiterHolder holder = getRateLimiter(methodName); + if (holder == null) { + return 0; + } + + int permitsToAcquire = Math.min(Math.max(permits, 1), holder.maxPermitsInTimeout); + // Resilience4j returns negative when the estimated wait exceeds the configured timeout. + // Otherwise the returned wait time is within meta_service_rpc_rate_limit_wait_timeout_ms. + long nanosToWait = holder.rateLimiter.reservePermission(permitsToAcquire); + if (nanosToWait < 0) { + throw new MetaServiceRateLimitException(methodName, + Config.meta_service_rpc_rate_limit_wait_timeout_ms); + } + if (nanosToWait == 0) { + return 0; + } + + long waitMs = TimeUnit.NANOSECONDS.toMillis(nanosToWait); + if (LOG.isDebugEnabled()) { + LOG.debug("meta service rpc rate limiter waits before acquiring permission, method: {}, permits: {}, " + + "original permits: {}, max permits in timeout: {}, limit for period: {}, " + + "burst seconds: {}, wait ms: {}", + methodName, permitsToAcquire, permits, holder.maxPermitsInTimeout, holder.limitForPeriod, + holder.burstSeconds, + waitMs); + } + long waitStartNs = System.nanoTime(); + try { + TimeUnit.NANOSECONDS.sleep(nanosToWait); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RpcException("", e.getMessage(), e); + } + long actualWaitNs = System.nanoTime() - waitStartNs; + if (MetricRepo.isInit && Config.isCloudMode()) { + CloudMetrics.META_SERVICE_RPC_RATE_LIMIT_WAIT_LATENCY.getOrAdd(methodName) + .update(TimeUnit.NANOSECONDS.toMillis(actualWaitNs)); + } + return actualWaitNs; + } + + private RateLimiterHolder getRateLimiter(String methodName) throws RpcException { + refreshConfigIfNeeded(); + RateLimitConfigSnapshot snapshot = currentSnapshot; + int qpsPerCore = methodQpsPerCore.getOrDefault(methodName, snapshot.defaultQpsPerCore); + if (qpsPerCore <= 0) { + rateLimiters.remove(methodName); + return null; + } + + int limitForPeriod = getLimitForPeriod(methodName, qpsPerCore, snapshot.burstSeconds); + int maxPermitsInTimeout = getMaxPermitsInTimeout(methodName, limitForPeriod, snapshot.burstSeconds, + snapshot.waitTimeoutMs); + RateLimiterHolder holder = rateLimiters.compute(methodName, (name, existingHolder) -> { + if (existingHolder != null && existingHolder.matches(limitForPeriod, snapshot.burstSeconds, + snapshot.waitTimeoutMs)) { + return existingHolder; + } + return new RateLimiterHolder(createRateLimiter(methodName, limitForPeriod, + snapshot.burstSeconds, snapshot.waitTimeoutMs), limitForPeriod, + maxPermitsInTimeout, snapshot.burstSeconds, snapshot.waitTimeoutMs); + }); + return holder; + } + + private void refreshConfigIfNeeded() throws RpcException { + RateLimitConfigSnapshot latestSnapshot = RateLimitConfigSnapshot.current(); + if (latestSnapshot.equals(currentSnapshot)) { + return; + } + + configLock.lock(); + try { + latestSnapshot = RateLimitConfigSnapshot.current(); + if (latestSnapshot.equals(currentSnapshot)) { + return; + } + validateConfig(latestSnapshot); + methodQpsPerCore = parseMethodQpsPerCore(latestSnapshot.qpsPerCoreConfig); + currentSnapshot = latestSnapshot; + } finally { + configLock.unlock(); + } + } + + private void validateConfig(RateLimitConfigSnapshot snapshot) throws RpcException { + try { + MetaServiceRpcRateLimitConfigValidator.validatePositive("meta_service_rpc_rate_limit_burst_seconds", + snapshot.burstSeconds); + MetaServiceRpcRateLimitConfigValidator.validateNonNegative("meta_service_rpc_rate_limit_wait_timeout_ms", + snapshot.waitTimeoutMs); + } catch (ConfigException e) { + throw new RpcException("", e.getMessage(), e); + } + } + + private Map parseMethodQpsPerCore(String config) throws RpcException { + try { + return MetaServiceRpcRateLimitConfigValidator.parseQpsPerCoreConfig(config); + } catch (ConfigException e) { + throw new RpcException("", "invalid meta_service_rpc_rate_limit_qps_per_core_config: " + config, e); + } + } + + private int getLimitForPeriod(String methodName, int qpsPerCore, int burstSeconds) throws RpcException { + long limitForPeriod = (long) qpsPerCore * CPU_CORES * burstSeconds; + if (limitForPeriod > Integer.MAX_VALUE) { + throw new RpcException("", "meta service rpc rate limit is too large, method: " + methodName + + ", qps per core: " + qpsPerCore + ", cpu cores: " + CPU_CORES + + ", burst seconds: " + burstSeconds); + } + return (int) limitForPeriod; + } + + private int getMaxPermitsInTimeout(String methodName, int limitForPeriod, int burstSeconds, long waitTimeoutMs) + throws RpcException { + if (waitTimeoutMs <= 0) { + return limitForPeriod; + } + + long refreshPeriodMs = TimeUnit.SECONDS.toMillis(burstSeconds); + // Keep a small margin from the timeout boundary, so a capped large batch does not fail repeatedly + // because of scheduling jitter or an already slow permit reservation path. + long maxPermitsInTimeout = Math.max(limitForPeriod, + (long) Math.floor(limitForPeriod * (double) waitTimeoutMs / refreshPeriodMs * 0.95D)); + if (maxPermitsInTimeout > Integer.MAX_VALUE) { + throw new RpcException("", "meta service rpc rate limit permits in timeout is too large, method: " + + methodName + ", limit for period: " + limitForPeriod + + ", burst seconds: " + burstSeconds + ", wait timeout ms: " + waitTimeoutMs); + } + return (int) maxPermitsInTimeout; + } + + private RateLimiter createRateLimiter(String methodName, int limitForPeriod, int burstSeconds, + long waitTimeoutMs) { + RateLimiterConfig rateLimiterConfig = RateLimiterConfig.custom() + .limitRefreshPeriod(Duration.ofSeconds(burstSeconds)) + .limitForPeriod(limitForPeriod) + .timeoutDuration(Duration.ofMillis(waitTimeoutMs)) + .build(); + return RateLimiter.of("meta_service_rpc_" + methodName, rateLimiterConfig); + } + + void reset() { + configLock.lock(); + try { + rateLimiters.clear(); + methodQpsPerCore = Collections.emptyMap(); + currentSnapshot = EMPTY_SNAPSHOT; + } finally { + configLock.unlock(); + } + } + + private static class RateLimiterHolder { + private final RateLimiter rateLimiter; + private final int limitForPeriod; + private final int maxPermitsInTimeout; + private final int burstSeconds; + private final long waitTimeoutMs; + + private RateLimiterHolder(RateLimiter rateLimiter, int limitForPeriod, int maxPermitsInTimeout, + int burstSeconds, long waitTimeoutMs) { + this.rateLimiter = rateLimiter; + this.limitForPeriod = limitForPeriod; + this.maxPermitsInTimeout = maxPermitsInTimeout; + this.burstSeconds = burstSeconds; + this.waitTimeoutMs = waitTimeoutMs; + } + + private boolean matches(int limitForPeriod, int burstSeconds, long waitTimeoutMs) { + return this.limitForPeriod == limitForPeriod && this.burstSeconds == burstSeconds + && this.waitTimeoutMs == waitTimeoutMs; + } + } + + private static class RateLimitConfigSnapshot { + private final int defaultQpsPerCore; + private final String qpsPerCoreConfig; + private final int burstSeconds; + private final long waitTimeoutMs; + + private RateLimitConfigSnapshot(int defaultQpsPerCore, String qpsPerCoreConfig, int burstSeconds, + long waitTimeoutMs) { + this.defaultQpsPerCore = defaultQpsPerCore; + this.qpsPerCoreConfig = qpsPerCoreConfig == null ? "" : qpsPerCoreConfig; + this.burstSeconds = burstSeconds; + this.waitTimeoutMs = waitTimeoutMs; + } + + private static RateLimitConfigSnapshot current() { + return new RateLimitConfigSnapshot(Config.meta_service_rpc_rate_limit_default_qps_per_core, + Config.meta_service_rpc_rate_limit_qps_per_core_config, + Config.meta_service_rpc_rate_limit_burst_seconds, + Config.meta_service_rpc_rate_limit_wait_timeout_ms); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RateLimitConfigSnapshot)) { + return false; + } + RateLimitConfigSnapshot that = (RateLimitConfigSnapshot) o; + return defaultQpsPerCore == that.defaultQpsPerCore && burstSeconds == that.burstSeconds + && waitTimeoutMs == that.waitTimeoutMs + && Objects.equals(qpsPerCoreConfig, that.qpsPerCoreConfig); + } + + @Override + public int hashCode() { + return Objects.hash(defaultQpsPerCore, qpsPerCoreConfig, burstSeconds, waitTimeoutMs); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java index 13b347694c94c9..b512871d85e0a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java @@ -23,8 +23,8 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.cloud.catalog.CloudColocatePlacement; +import org.apache.doris.cloud.catalog.CloudComputeGroupMeta; import org.apache.doris.cloud.catalog.CloudEnv; -import org.apache.doris.cloud.catalog.ComputeGroup; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.ClusterPB; import org.apache.doris.cloud.proto.Cloud.InstanceInfoPB; @@ -99,8 +99,8 @@ public class CloudSystemInfoService extends SystemInfoService { // clusterName -> clusterId protected Map clusterNameToId = new ConcurrentHashMap<>(); - // clusterId -> ComputeGroup - protected Map computeGroupIdToComputeGroup = new ConcurrentHashMap<>(); + // clusterId -> CloudComputeGroupMeta + protected Map computeGroupIdToComputeGroup = new ConcurrentHashMap<>(); private final Map colocatePlacementCache = new ConcurrentHashMap<>(); @@ -260,7 +260,7 @@ public void renameVirtualClusterInfoFromMapsNoLock(String clusterId, String oldC clusterNameToId.remove(oldClusterName); } - public ComputeGroup getComputeGroupByName(String computeGroupName) { + public CloudComputeGroupMeta getComputeGroupByName(String computeGroupName) { // rlock guards the compound name->id->group lookup: writers (add/remove/rename) // update both maps under wlock, and the read must observe a consistent snapshot // so callers like getPhysicalCluster don't transiently see a virtual group name @@ -355,7 +355,7 @@ public String resolveClusterIdByName(String cluster) throws ComputeGroupExceptio return getCloudClusterIdByName(cluster); } - public ComputeGroup getComputeGroupById(String computeGroupId) { + public CloudComputeGroupMeta getComputeGroupById(String computeGroupId) { try { rlock.lock(); return computeGroupIdToComputeGroup.get(computeGroupId); @@ -364,7 +364,7 @@ public ComputeGroup getComputeGroupById(String computeGroupId) { } } - public void addComputeGroup(String computeGroupId, ComputeGroup computeGroup) { + public void addComputeGroup(String computeGroupId, CloudComputeGroupMeta computeGroup) { LOG.debug("add id {} computeGroupIdToComputeGroup : {} ", computeGroupId, computeGroupIdToComputeGroup); try { wlock.lock(); @@ -376,8 +376,8 @@ public void addComputeGroup(String computeGroupId, ComputeGroup computeGroup) { } public boolean isStandByComputeGroup(String clusterName) { - List virtualGroups = getComputeGroups(true); - for (ComputeGroup vcg : virtualGroups) { + List virtualGroups = getComputeGroups(true); + for (CloudComputeGroupMeta vcg : virtualGroups) { if (vcg.getPolicy().getStandbyComputeGroup().equals(clusterName)) { return true; } @@ -385,7 +385,7 @@ public boolean isStandByComputeGroup(String clusterName) { return false; } - public List getComputeGroups(boolean virtual) { + public List getComputeGroups(boolean virtual) { LOG.debug("get virtual {} computeGroupIdToComputeGroup : {} ", virtual, computeGroupIdToComputeGroup); try { rlock.lock(); @@ -404,7 +404,7 @@ public List getComputeGroups(boolean virtual) { public String ownedByVirtualComputeGroup(String computeGroupName) { try { rlock.lock(); - for (ComputeGroup vcg : getComputeGroups(true)) { + for (CloudComputeGroupMeta vcg : getComputeGroups(true)) { if (computeGroupName.equals(vcg.getPolicy().getActiveComputeGroup())) { return vcg.getName(); } @@ -433,7 +433,7 @@ public void removeComputeGroup(String computeGroupId, String computeGroupName) { } public void renameVirtualComputeGroup(String computeGroupId, String oldComputeGroupName, - ComputeGroup newComputeGroup) { + CloudComputeGroupMeta newComputeGroup) { try { wlock.lock(); computeGroupIdToComputeGroup.put(computeGroupId, newComputeGroup); @@ -587,7 +587,8 @@ public void updateCloudClusterMapNoLock(List toAdd, List toDel clusterNameToId.put(clusterName, clusterId); // add to computeGroupIdToComputeGroup - ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta( + clusterId, clusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); addComputeGroup(clusterId, cg); List be = clusterIdToBackend.get(clusterId); @@ -685,7 +686,7 @@ public synchronized void updateFrontends(List toAdd, List to } } - public static void updateFileCacheJobIds(ComputeGroup cg, List jobIds) { + public static void updateFileCacheJobIds(CloudComputeGroupMeta cg, List jobIds) { Cloud.ClusterPolicy policy = Cloud.ClusterPolicy.newBuilder() .setType(Cloud.ClusterPolicy.PolicyType.ActiveStandby) .addAllCacheWarmupJobids(jobIds).build(); @@ -727,7 +728,7 @@ enum PolicyType { } */ - private void switchActiveStandby(ComputeGroup cg, String active, String standby) { + private void switchActiveStandby(CloudComputeGroupMeta cg, String active, String standby) { Cloud.ClusterPolicy policy = cg.getPolicy().toPb().toBuilder() .clearStandbyClusterNames() .addStandbyClusterNames(active) @@ -1060,7 +1061,7 @@ public List getBackendsByClusterId(final String clusterId) { } public String getPhysicalCluster(String clusterName) { - ComputeGroup cg = getComputeGroupByName(clusterName); + CloudComputeGroupMeta cg = getComputeGroupByName(clusterName); if (cg == null) { return clusterName; } @@ -1069,11 +1070,11 @@ public String getPhysicalCluster(String clusterName) { return clusterName; } - ComputeGroup.Policy policy = cg.getPolicy(); + CloudComputeGroupMeta.Policy policy = cg.getPolicy(); // todo check policy String acgName = policy.getActiveComputeGroup(); if (acgName != null) { - ComputeGroup acg = getComputeGroupByName(acgName); + CloudComputeGroupMeta acg = getComputeGroupByName(acgName); if (acg != null) { if (isComputeGroupAvailable(acgName, policy.getUnhealthyNodeThresholdPercent())) { acg.setUnavailableSince(-1); @@ -1089,11 +1090,11 @@ public String getPhysicalCluster(String clusterName) { String scgName = policy.getStandbyComputeGroup(); if (scgName != null) { - ComputeGroup scg = getComputeGroupByName(scgName); + CloudComputeGroupMeta scg = getComputeGroupByName(scgName); if (scg != null) { if (isComputeGroupAvailable(scgName, policy.getUnhealthyNodeThresholdPercent())) { scg.setUnavailableSince(-1); - ComputeGroup acg = getComputeGroupByName(acgName); + CloudComputeGroupMeta acg = getComputeGroupByName(acgName); if (acg == null || System.currentTimeMillis() - acg.getUnavailableSince() > policy.getFailoverFailureThreshold() * Config.heartbeat_interval_second * 1000) { switchActiveStandby(cg, acgName, scgName); @@ -1378,7 +1379,7 @@ public Map> getCloudClusterIdToBackend(boolean needVirtual clusterId, computeGroupIdToComputeGroup); continue; } - ComputeGroup computeGroup = computeGroupIdToComputeGroup.get(clusterId); + CloudComputeGroupMeta computeGroup = computeGroupIdToComputeGroup.get(clusterId); if (!needVirtual && computeGroup.isVirtual()) { continue; } @@ -1421,7 +1422,7 @@ public Map getCloudClusterNameToId(boolean needVirtual) { try { for (Map.Entry nameAndId : clusterNameToId.entrySet()) { String clusterId = nameAndId.getValue(); - ComputeGroup computeGroup = computeGroupIdToComputeGroup.get(clusterId); + CloudComputeGroupMeta computeGroup = computeGroupIdToComputeGroup.get(clusterId); if (computeGroup == null) { LOG.warn("cant find clusterId {} in computeGroupIdToComputeGroup {}", clusterId, computeGroupIdToComputeGroup); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 0749d38f3111c4..1cf69114d0981e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -1841,7 +1841,7 @@ public void abortTransaction(Long dbId, Long transactionId, String reason, AbortTxnResponse abortTxnResponse = null; try { - abortTxnResponse = abortTransactionImpl(dbId, transactionId, reason, null); + abortTxnResponse = abortTransactionImpl(dbId, transactionId, reason, txnCommitAttachment); } finally { handleAfterAbort(abortTxnResponse, txnCommitAttachment, transactionId, callbackInfo.first, callbackInfo.second); @@ -1905,6 +1905,10 @@ private AbortTxnResponse abortTransactionImpl(Long dbId, Long transactionId, Str builder.setTxnId(transactionId); builder.setReason(reason); builder.setCloudUniqueId(Config.cloud_unique_id); + if (txnCommitAttachment instanceof RLTaskTxnCommitAttachment) { + builder.setCommitAttachment(TxnUtil.rlTaskTxnCommitAttachmentToPb( + (RLTaskTxnCommitAttachment) txnCommitAttachment)); + } final AbortTxnRequest abortTxnRequest = builder.build(); AbortTxnResponse abortTxnResponse = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/TxnUtil.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/TxnUtil.java index f29aa5294b3781..fceb16d91add52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/TxnUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/TxnUtil.java @@ -256,6 +256,9 @@ public static TxnCommitAttachmentPB rlTaskTxnCommitAttachmentToPb(RLTaskTxnCommi if (rtTaskTxnCommitAttachment.getErrorLogUrl() != null) { builder.setErrorLogUrl(rtTaskTxnCommitAttachment.getErrorLogUrl()); } + if (rtTaskTxnCommitAttachment.getFirstErrorMsg() != null) { + builder.setFirstErrorMsg(rtTaskTxnCommitAttachment.getFirstErrorMsg()); + } attachementBuilder.setRlTaskTxnCommitAttachment(builder.build()); return attachementBuilder.build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index 0cf43bb1757cca..9e40042d32d754 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -107,11 +107,18 @@ public static void checkColumnName(String columnName) throws AnalysisException { } public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException { + checkColumnNameBypassSystemColumnPrefix(columnName); + checkColumnNamePrefix(columnName, Column.SHADOW_NAME_PREFIX); + } + + /** + * Check column name syntax without applying Doris top-level hidden/shadow column prefix rules. + */ + public static void checkColumnNameBypassSystemColumnPrefix(String columnName) throws AnalysisException { if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, columnName, getColumnNameRegex()); } - checkColumnNamePrefix(columnName, Column.SHADOW_NAME_PREFIX); } private static void checkColumnNamePrefix(String columnName, String prefix) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java index 58fc49ae4c3460..7b7803144115f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java @@ -164,6 +164,7 @@ public static void getFrontendsInfo(Env env, List> infos, String cu // To indicate which FE we currently connected info.add(fe.getHost().equals(selfNode) ? "Yes" : "No"); info.add(TimeUtils.longToTimeString(fe.getLiveSince())); + info.add(fe.getLocalResourceGroup()); infos.add(info); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java index 124be52b05b078..7011e28df7eb08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java @@ -106,6 +106,7 @@ public class SummaryProfile { public static final String FETCH_RESULT_TIME = "Fetch Result Time"; public static final String WRITE_RESULT_TIME = "Write Result Time"; public static final String GET_META_VERSION_TIME = "Get Meta Version Time"; + public static final String GET_META_VERSION_RATE_LIMIT_WAIT_TIME = "Get Meta Version Rate Limit Wait Time"; public static final String GET_PARTITION_VERSION_TIME = "Get Partition Version Time"; public static final String GET_PARTITION_VERSION_COUNT = "Get Partition Version Count"; public static final String GET_PARTITION_VERSION_BY_HAS_DATA_COUNT = "Get Partition Version Count (hasData)"; @@ -218,6 +219,7 @@ public boolean isWarmup() { PAIMON_SCAN_METRICS, NEREIDS_DISTRIBUTE_TIME, GET_META_VERSION_TIME, + GET_META_VERSION_RATE_LIMIT_WAIT_TIME, GET_PARTITION_VERSION_TIME, GET_PARTITION_VERSION_BY_HAS_DATA_COUNT, GET_PARTITION_VERSION_COUNT, @@ -275,6 +277,7 @@ public boolean isWarmup() { .put(CREATE_SCAN_RANGE_TIME, 2) .put(ICEBERG_SCAN_METRICS, 3) .put(PAIMON_SCAN_METRICS, 3) + .put(GET_META_VERSION_RATE_LIMIT_WAIT_TIME, 1) .put(GET_PARTITION_VERSION_TIME, 1) .put(GET_PARTITION_VERSION_COUNT, 1) .put(GET_PARTITION_VERSION_BY_HAS_DATA_COUNT, 1) @@ -403,6 +406,8 @@ public boolean isWarmup() { private long getTableVersionTime = 0; @SerializedName(value = "getTableVersionCount") private long getTableVersionCount = 0; + @SerializedName(value = "getMetaVersionRateLimitWaitTime") + private long getMetaVersionRateLimitWaitTime = 0; @SerializedName(value = "transactionCommitBeginTime") private long transactionCommitBeginTime = -1; @SerializedName(value = "transactionCommitEndTime") @@ -678,6 +683,8 @@ private void updateExecutionSummaryProfile() { if (Config.isCloudMode()) { executionSummaryProfile.addInfoString(GET_META_VERSION_TIME, getPrettyGetMetaVersionTime()); + executionSummaryProfile.addInfoString(GET_META_VERSION_RATE_LIMIT_WAIT_TIME, + getPrettyGetMetaVersionRateLimitWaitTime()); executionSummaryProfile.addInfoString(GET_PARTITION_VERSION_TIME, getPrettyGetPartitionVersionTime()); executionSummaryProfile.addInfoString(GET_PARTITION_VERSION_COUNT, getPrettyGetPartitionVersionCount()); executionSummaryProfile.addInfoString(GET_PARTITION_VERSION_BY_HAS_DATA_COUNT, @@ -882,6 +889,10 @@ public void addGetTableVersionTime(long ns) { this.getTableVersionCount += 1; } + public void addGetMetaVersionRateLimitWaitTime(long ns) { + this.getMetaVersionRateLimitWaitTime += ns; + } + public void incGetPartitionVersionByHasDataCount() { this.getPartitionVersionByHasDataCount += 1; } @@ -1065,6 +1076,13 @@ private String getPrettyGetMetaVersionTime() { return RuntimeProfile.printCounter(getMetaVersionTime, TUnit.TIME_NS); } + private String getPrettyGetMetaVersionRateLimitWaitTime() { + if (getMetaVersionRateLimitWaitTime == 0) { + return "N/A"; + } + return RuntimeProfile.printCounter(getMetaVersionRateLimitWaitTime, TUnit.TIME_NS); + } + private String getPrettyGetPartitionVersionTime() { if (getPartitionVersionTime == 0) { return "N/A"; @@ -1095,8 +1113,8 @@ private String getPrettyGetTableVersionCount() { return RuntimeProfile.printCounter(getTableVersionCount, TUnit.UNIT); } - public long getGetPartitionVersionTime() { - return getPartitionVersionTime; + public long getGetPartitionVersionTimeMs() { + return TimeUnit.NANOSECONDS.toMillis(getPartitionVersionTime); } public long getGetPartitionVersionCount() { @@ -1107,14 +1125,18 @@ public long getGetPartitionVersionByHasDataCount() { return getPartitionVersionByHasDataCount; } - public long getGetTableVersionTime() { - return getTableVersionTime; + public long getGetTableVersionTimeMs() { + return TimeUnit.NANOSECONDS.toMillis(getTableVersionTime); } public long getGetTableVersionCount() { return getTableVersionCount; } + public long getGetMetaVersionRateLimitWaitTime() { + return getMetaVersionRateLimitWaitTime; + } + private String getPrettyTime(long end, long start, TUnit unit) { if (start == -1 || end == -1) { return "N/A"; @@ -1383,13 +1405,17 @@ public String getPlanTime() { } public String getMetaTime() { - return "{" - + "\"get_partition_version_time_ms\"" + ":" + this.getGetPartitionVersionTime() + "," + String metaTime = "{" + + "\"get_partition_version_time_ms\"" + ":" + this.getGetPartitionVersionTimeMs() + "," + "\"get_partition_version_count_has_data\"" + ":" + this.getGetPartitionVersionByHasDataCount() + "," + "\"get_partition_version_count\"" + ":" + this.getGetPartitionVersionCount() + "," - + "\"get_table_version_time_ms\"" + ":" + this.getGetTableVersionTime() + "," - + "\"get_table_version_count\"" + ":" + this.getGetTableVersionCount() - + "}"; + + "\"get_table_version_time_ms\"" + ":" + this.getGetTableVersionTimeMs() + "," + + "\"get_table_version_count\"" + ":" + this.getGetTableVersionCount(); + if (this.getGetMetaVersionRateLimitWaitTime() > 0) { + metaTime += ",\"get_meta_version_rate_limit_wait_time_ms\"" + ":" + + TimeUnit.NANOSECONDS.toMillis(this.getGetMetaVersionRateLimitWaitTime()); + } + return metaTime + "}"; } public String getScheduleTime() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java index feb88abbb1cec5..f1fa2f87d4012a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java @@ -20,8 +20,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.cloud.security.SecurityChecker; import org.apache.doris.common.Config; +import org.apache.doris.httpv2.meta.MetaBaseAction; import org.apache.doris.system.SystemInfoService.HostInfo; +import com.google.common.base.Strings; import com.google.common.collect.Maps; import org.apache.http.conn.ssl.NoopHostnameVerifier; @@ -50,6 +52,10 @@ public static HttpURLConnection getConnectionWithNodeIdent(String request) throw HostInfo selfNode = Env.getServingEnv().getSelfNode(); conn.setRequestProperty(Env.CLIENT_NODE_HOST_KEY, selfNode.getHost()); conn.setRequestProperty(Env.CLIENT_NODE_PORT_KEY, selfNode.getPort() + ""); + String token = Config.fe_meta_auth_token; + if (!Strings.isNullOrEmpty(token)) { + conn.setRequestProperty(MetaBaseAction.TOKEN, token); + } return conn; } catch (Exception e) { throw new IOException(e); @@ -65,6 +71,10 @@ public static Map getNodeIdentHeaders() throws IOException { HostInfo selfNode = Env.getServingEnv().getSelfNode(); headers.put(Env.CLIENT_NODE_HOST_KEY, selfNode.getHost()); headers.put(Env.CLIENT_NODE_PORT_KEY, selfNode.getPort() + ""); + String token = Config.fe_meta_auth_token; + if (!Strings.isNullOrEmpty(token)) { + headers.put(MetaBaseAction.TOKEN, token); + } return headers; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/InternalHttpsUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/InternalHttpsUtils.java index f0c7f7db7ddf05..47f0cebd9a3732 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/InternalHttpsUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/InternalHttpsUtils.java @@ -30,15 +30,17 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.security.KeyStore; +import java.security.cert.Certificate; +import java.util.Collections; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManagerFactory; /** - * Utility for creating SSL-aware HTTP clients for internal FE-to-FE communication. + * SSL-aware HTTP client for internal FE communication (loopback and FE-to-FE). * - *

Builds an {@link SSLContext} from the configured CA truststore once and caches it. - * Hostname verification is disabled for IP-based intra-cluster connections. - * Certificate rotation requires a FE restart. + *

Trusts this FE's own HTTPS keystore ({@code Config.key_store_path}), not + * {@code Config.mysql_ssl_default_ca_certificate} (a separate store for MySQL SSL). Trusts every + * cert in the keystore's chains, so a bundled CA also validates other nodes' certs it signed. */ public class InternalHttpsUtils { private static volatile SSLContext cachedSslContext = null; @@ -46,7 +48,7 @@ public class InternalHttpsUtils { private static final Logger LOG = LogManager.getLogger(InternalHttpsUtils.class); /** - * Returns the cached SSLContext, building it from the configured truststore on first call. + * Returns the cached SSLContext, building it from the FE's own HTTPS keystore on first call. */ public static SSLContext getSslContext() { if (cachedSslContext == null) { @@ -61,30 +63,48 @@ public static SSLContext getSslContext() { private static SSLContext buildSslContext() { try { - // The same CA signs all Doris TLS certs (FE HTTPS + MySQL SSL), so mysql_ssl_default_ca_certificate - // is the correct trust anchor for FE-to-FE HTTPS. Hostname verification is skipped for IP-based comms. - KeyStore trustStore = KeyStore.getInstance(Config.ssl_trust_store_type); - try (InputStream stream = Files.newInputStream( - Paths.get(Config.mysql_ssl_default_ca_certificate))) { - trustStore.load(stream, Config.mysql_ssl_default_ca_certificate_password.toCharArray()); + KeyStore keyStore = KeyStore.getInstance(Config.key_store_type); + try (InputStream stream = Files.newInputStream(Paths.get(Config.key_store_path))) { + keyStore.load(stream, Config.key_store_password.toCharArray()); } TrustManagerFactory tmf = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm()); - tmf.init(trustStore); + tmf.init(buildTrustStore(keyStore)); SSLContext sslContext = SSLContext.getInstance("TLS"); sslContext.init(null, tmf.getTrustManagers(), null); return sslContext; } catch (Exception e) { - LOG.error("Failed to build SSLContext from truststore: {}", - Config.mysql_ssl_default_ca_certificate, e); + LOG.error("Failed to build SSLContext from FE HTTPS keystore: {}", + Config.key_store_path, e); throw new RuntimeException( - "Failed to build SSLContext from truststore: " - + Config.mysql_ssl_default_ca_certificate, e); + "Failed to build SSLContext from FE HTTPS keystore: " + + Config.key_store_path, e); } } + // Extracts every cert in every chain, since KeyStore.getCertificate() only returns the leaf. + private static KeyStore buildTrustStore(KeyStore keyStore) throws Exception { + KeyStore trustStore = KeyStore.getInstance(Config.key_store_type); + trustStore.load(null, null); + int certIndex = 0; + for (String alias : Collections.list(keyStore.aliases())) { + Certificate[] chain = keyStore.getCertificateChain(alias); + if (chain == null) { + Certificate cert = keyStore.getCertificate(alias); + if (cert != null) { + trustStore.setCertificateEntry("cert-" + certIndex++, cert); + } + continue; + } + for (Certificate cert : chain) { + trustStore.setCertificateEntry("cert-" + certIndex++, cert); + } + } + return trustStore; + } + /** * Returns an HTTP client configured with the FE CA truststore and hostname verification disabled. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index c598ef520f5ef6..5c4decd93a30b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.TableIf; @@ -249,6 +250,15 @@ default void addColumn(TableIf table, Column column, ColumnPosition columnPositi throw new UserException("Not support add column operation"); } + default void addColumn(TableIf table, ColumnPath columnPath, Column column, ColumnPosition columnPosition) + throws UserException { + if (!columnPath.isNested()) { + addColumn(table, column, columnPosition); + return; + } + throw new UserException("Not support nested add column operation"); + } + default void addColumns(TableIf table, List columns) throws UserException { throw new UserException("Not support add columns operation"); } @@ -257,14 +267,47 @@ default void dropColumn(TableIf table, String name) throws UserException { throw new UserException("Not support drop column operation"); } + default void dropColumn(TableIf table, ColumnPath columnPath) throws UserException { + if (!columnPath.isNested()) { + dropColumn(table, columnPath.getTopLevelName()); + return; + } + throw new UserException("Not support nested drop column operation"); + } + default void renameColumn(TableIf table, String oldName, String newName) throws UserException { throw new UserException("Not support rename column operation"); } + default void renameColumn(TableIf table, ColumnPath columnPath, String newName) throws UserException { + if (!columnPath.isNested()) { + renameColumn(table, columnPath.getTopLevelName(), newName); + return; + } + throw new UserException("Not support nested rename column operation"); + } + default void modifyColumn(TableIf table, Column column, ColumnPosition columnPosition) throws UserException { throw new UserException("Not support update column operation"); } + default void modifyColumn(TableIf table, ColumnPath columnPath, Column column, ColumnPosition columnPosition) + throws UserException { + if (!columnPath.isNested()) { + modifyColumn(table, column, columnPosition); + return; + } + throw new UserException("Not support nested modify column operation"); + } + + default void modifyColumnComment(TableIf table, String name, String comment) throws UserException { + modifyColumnComment(table, ColumnPath.of(name), comment); + } + + default void modifyColumnComment(TableIf table, ColumnPath columnPath, String comment) throws UserException { + throw new UserException("Not support modify column comment operation"); + } + default void reorderColumns(TableIf table, List newOrder) throws UserException { throw new UserException("Not support reorder columns operation"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index a742a50f9a62d6..341477c9911253 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; @@ -1536,6 +1537,12 @@ private void logRefreshExternalTable(ExternalTable dorisTable, long updateTime) @Override public void addColumn(TableIf dorisTable, Column column, ColumnPosition position) throws UserException { + addColumn(dorisTable, ColumnPath.of(column.getName()), column, position); + } + + @Override + public void addColumn(TableIf dorisTable, ColumnPath columnPath, Column column, ColumnPosition position) + throws UserException { makeSureInitialized(); Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); ExternalTable externalTable = (ExternalTable) dorisTable; @@ -1544,11 +1551,11 @@ public void addColumn(TableIf dorisTable, Column column, ColumnPosition position } try { long updateTime = System.currentTimeMillis(); - metadataOps.addColumn(externalTable, column, position, updateTime); + metadataOps.addColumn(externalTable, columnPath, column, position, updateTime); logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to add column {} to table {}.{} in catalog {}", - column.getName(), externalTable.getDbName(), externalTable.getName(), getName(), e); + columnPath.getFullPath(), externalTable.getDbName(), externalTable.getName(), getName(), e); throw e; } } @@ -1574,6 +1581,11 @@ public void addColumns(TableIf dorisTable, List columns) throws UserExce @Override public void dropColumn(TableIf dorisTable, String columnName) throws UserException { + dropColumn(dorisTable, ColumnPath.of(columnName)); + } + + @Override + public void dropColumn(TableIf dorisTable, ColumnPath columnPath) throws UserException { makeSureInitialized(); Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); ExternalTable externalTable = (ExternalTable) dorisTable; @@ -1582,17 +1594,22 @@ public void dropColumn(TableIf dorisTable, String columnName) throws UserExcepti } try { long updateTime = System.currentTimeMillis(); - metadataOps.dropColumn(externalTable, columnName, updateTime); + metadataOps.dropColumn(externalTable, columnPath, updateTime); logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { LOG.warn("Failed to drop column {} from table {}.{} in catalog {}", - columnName, externalTable.getDbName(), externalTable.getName(), getName(), e); + columnPath.getFullPath(), externalTable.getDbName(), externalTable.getName(), getName(), e); throw e; } } @Override public void renameColumn(TableIf dorisTable, String oldName, String newName) throws UserException { + renameColumn(dorisTable, ColumnPath.of(oldName), newName); + } + + @Override + public void renameColumn(TableIf dorisTable, ColumnPath columnPath, String newName) throws UserException { makeSureInitialized(); Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); ExternalTable externalTable = (ExternalTable) dorisTable; @@ -1601,11 +1618,11 @@ public void renameColumn(TableIf dorisTable, String oldName, String newName) thr } try { long updateTime = System.currentTimeMillis(); - metadataOps.renameColumn(externalTable, oldName, newName, updateTime); + metadataOps.renameColumn(externalTable, columnPath, newName, updateTime); logRefreshExternalTable(externalTable, updateTime); } catch (Exception e) { - LOG.warn("Failed to rename column {} to {} in table {}.{} in catalog {}", - oldName, newName, externalTable.getDbName(), externalTable.getName(), getName(), e); + LOG.warn("Failed to rename column {} to {} in table {}.{} in catalog {}", columnPath.getFullPath(), + newName, externalTable.getDbName(), externalTable.getName(), getName(), e); throw e; } } @@ -1629,6 +1646,45 @@ public void modifyColumn(TableIf dorisTable, Column column, ColumnPosition colum } } + @Override + public void modifyColumn(TableIf dorisTable, ColumnPath columnPath, Column column, ColumnPosition columnPosition) + throws UserException { + makeSureInitialized(); + Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); + ExternalTable externalTable = (ExternalTable) dorisTable; + if (metadataOps == null) { + throw new DdlException("Modify column operation is not supported for catalog: " + getName()); + } + try { + long updateTime = System.currentTimeMillis(); + metadataOps.modifyColumn(externalTable, columnPath, column, columnPosition, updateTime); + logRefreshExternalTable(externalTable, updateTime); + } catch (Exception e) { + LOG.warn("Failed to modify column {} in table {}.{} in catalog {}", + columnPath.getFullPath(), externalTable.getDbName(), externalTable.getName(), getName(), e); + throw e; + } + } + + @Override + public void modifyColumnComment(TableIf dorisTable, ColumnPath columnPath, String comment) throws UserException { + makeSureInitialized(); + Preconditions.checkState(dorisTable instanceof ExternalTable, dorisTable.getName()); + ExternalTable externalTable = (ExternalTable) dorisTable; + if (metadataOps == null) { + throw new DdlException("Modify column comment operation is not supported for catalog: " + getName()); + } + try { + long updateTime = System.currentTimeMillis(); + metadataOps.modifyColumnComment(externalTable, columnPath, comment, updateTime); + logRefreshExternalTable(externalTable, updateTime); + } catch (Exception e) { + LOG.warn("Failed to modify column comment {} in table {}.{} in catalog {}", + columnPath.getFullPath(), externalTable.getDbName(), externalTable.getName(), getName(), e); + throw e; + } + } + @Override public void reorderColumns(TableIf dorisTable, List newOrder) throws UserException { makeSureInitialized(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 6a986da45303d6..37dbf4cdd390ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -124,8 +124,7 @@ private static TStructField getExternalSchemaForPrunedColumn(List columns, Map> nameMapping) { - initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, Collections.emptyMap()); + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, List columns, Map> nameMapping, Map base64InitialDefaults) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), base64InitialDefaults); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { params.setCurrentSchemaId(schemaId); TSchema tSchema = new TSchema(); tSchema.setSchemaId(schemaId); - tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping, base64InitialDefaults)); + tSchema.setRootField(getExternalSchemaForAllColumn( + columns, nameMapping, hasNameMapping, base64InitialDefaults)); params.addToHistorySchemaInfo(tSchema); } private static TStructField getExternalSchemaForAllColumn(List columns, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TStructField structField = new TStructField(); for (Column child : columns) { TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( - child.getType(), child, nameMapping, base64InitialDefaults)); + child.getType(), child, nameMapping, hasNameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } return structField; @@ -161,11 +170,13 @@ private static TStructField getExternalSchemaForAllColumn(List columns, private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping) { - return getExternalSchema(columnType, dorisColumn, nameMapping, Collections.emptyMap()); + return getExternalSchema(columnType, dorisColumn, nameMapping, + nameMapping != null && !nameMapping.isEmpty(), Collections.emptyMap()); } private static TField getExternalSchema(Type columnType, Column dorisColumn, - Map> nameMapping, Map base64InitialDefaults) { + Map> nameMapping, boolean hasNameMapping, + Map base64InitialDefaults) { TField root = new TField(); root.setName(dorisColumn.getName()); root.setId(dorisColumn.getUniqueId()); @@ -178,9 +189,12 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, root.setInitialDefaultValue(dorisColumn.getDefaultValue()); } - if (nameMapping != null && nameMapping.containsKey(dorisColumn.getUniqueId())) { - // for iceberg set name mapping. - root.setNameMapping(new ArrayList<>(nameMapping.get(dorisColumn.getUniqueId()))); + if (hasNameMapping) { + // The explicit capability keeps old-FE plans on legacy fallback while making an empty + // per-field mapping authoritative for plans produced by a compatible FE. + root.setNameMapping(new ArrayList<>( + nameMapping.getOrDefault(dorisColumn.getUniqueId(), Collections.emptyList()))); + root.setNameMappingIsAuthoritative(true); } TNestedField nestedField = new TNestedField(); @@ -199,7 +213,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); Column subColumn = subNameToSubColumn.get(subField.getName()); fieldPtr.setFieldPtr(getExternalSchema( - subField.getType(), subColumn, nameMapping, base64InitialDefaults)); + subField.getType(), subColumn, nameMapping, hasNameMapping, + base64InitialDefaults)); structField.addToFields(fieldPtr); } @@ -212,7 +227,7 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); root.setNestedField(nestedField); @@ -223,13 +238,13 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TFieldPtr keyPtr = new TFieldPtr(); keyPtr.setFieldPtr(getExternalSchema( dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setKeyField(keyPtr); TFieldPtr valuePtr = new TFieldPtr(); valuePtr.setFieldPtr(getExternalSchema( dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping, - base64InitialDefaults)); + hasNameMapping, base64InitialDefaults)); mapField.setValueField(valuePtr); nestedField.setMapField(mapField); root.setNestedField(nestedField); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 55da4f30ad7043..b1dc260701eb03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -181,7 +181,7 @@ protected void initSchemaParams() throws UserException { slotInfo.setSlotId(slot.getId().asInt()); TColumnCategory category = classifyColumn(slot, partitionKeys); slotInfo.setCategory(category); - slotInfo.setIsFileSlot(category == TColumnCategory.REGULAR || category == TColumnCategory.GENERATED); + slotInfo.setIsFileSlot(isFileSlot(category)); params.addToRequiredSlots(slotInfo); } setDefaultValueExprs(getTargetTable(), destSlotDescByName, null, params, false); @@ -210,7 +210,7 @@ private void updateRequiredSlots() throws UserException { slotInfo.setSlotId(slot.getId().asInt()); TColumnCategory category = classifyColumn(slot, partitionKeys); slotInfo.setCategory(category); - slotInfo.setIsFileSlot(category == TColumnCategory.REGULAR || category == TColumnCategory.GENERATED); + slotInfo.setIsFileSlot(isFileSlot(category)); params.addToRequiredSlots(slotInfo); } // Update required slots and column_idxs in scanRangeLocations. @@ -228,6 +228,10 @@ protected TColumnCategory classifyColumn(SlotDescriptor slot, List parti return TColumnCategory.REGULAR; } + protected boolean isFileSlot(TColumnCategory category) { + return category == TColumnCategory.REGULAR || category == TColumnCategory.GENERATED; + } + public void setTableSample(TableSample tSample) { this.tableSample = tSample; } @@ -476,8 +480,9 @@ private TScanRangeLocations splitToScanRange( pathPartitionKeys, partitionValuesFromPath.getIsNull()); TFileCompressType fileCompressType = getFileCompressType(fileSplit); rangeDesc.setCompressType(fileCompressType); - // set file format type, and the type might fall back to native format in setScanParams - rangeDesc.setFormatType(getFileFormatType()); + // Seed connector-specific setup with the scan-level default. A connector may then + // override it with the actual format carried by an individual split. + rangeDesc.setFormatType(params.getFormatType()); setScanParams(rangeDesc, fileSplit); rangeDesc.setFileCacheAdmission(admissionResult); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index 466dad11d5dc49..53b0d11c5c7283 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -88,6 +88,8 @@ public FileScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, @Override protected void toThrift(TPlanNode planNode) { planNode.setPushDownAggTypeOpt(pushDownAggNoGroupingOp); + planNode.setPushDownCountSlotIds( + pushDownCountSlotIds.stream().map(id -> id.asInt()).collect(Collectors.toList())); planNode.setNodeType(TPlanNodeType.FILE_SCAN_NODE); TFileScanNode fileScanNode = new TFileScanNode(); @@ -103,6 +105,21 @@ protected void setPushDownCount(long count) { tableLevelRowCount = count; } + /** + * Return whether FE may replace real table-format splits with metadata COUNT splits. + * + *

The aggregate opcode alone is insufficient because both {@code COUNT(*)} and + * {@code COUNT(col)} use {@link TPushAggOp#COUNT}. The semantic argument list distinguishes + * them: it is empty only for {@code COUNT(*)}/{@code COUNT(1)}. For example, if an Iceberg + * table has 100 data files, retaining one representative split is correct for a snapshot + * {@code COUNT(*)}. Doing that for {@code COUNT(required_col)} is unsafe: BE deliberately + * falls back to reading the column, but it would then see only the representative file and + * undercount the table. + */ + protected boolean isTableLevelCountStarPushdown() { + return pushDownAggNoGroupingOp == TPushAggOp.COUNT && pushDownCountSlotIds.isEmpty(); + } + private long getPushDownCount() { return tableLevelRowCount; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java index 4ff3d85f3b9860..49d9a0de90f866 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java @@ -223,6 +223,21 @@ public boolean accept(String fileName) { public static FileCacheValue getAcidState(FileSystem fileSystem, HivePartition partition, Map txnValidIds, Map storagePropertiesMap, boolean isFullAcid) throws Exception { + return getAcidState(fileSystem, partition, txnValidIds, storagePropertiesMap, isFullAcid, + partition.getPath()); + } + + /** + * Variant taking the partition location already normalized by the same {@code LocationPath} + * resolution that selected {@code fileSystem}. Concrete filesystems only accept their native + * schemes, so when the legacy cross-scheme fallback fires (e.g. a {@code cos://} HMS location + * served by s3.* storage properties) the raw partition path must not be globbed directly; + * using the normalized path keeps base/delta listing and the BE-facing AcidInfo locations + * consistent with the non-transactional listing path. + */ + public static FileCacheValue getAcidState(FileSystem fileSystem, HivePartition partition, + Map txnValidIds, Map storagePropertiesMap, + boolean isFullAcid, String partitionPath) throws Exception { // Ref: https://issues.apache.org/jira/browse/HIVE-18192 // Readers should use the combination of ValidTxnList and ValidWriteIdList(Table) for snapshot isolation. @@ -248,8 +263,7 @@ public static FileCacheValue getAcidState(FileSystem fileSystem, HivePartition p throw new RuntimeException("Miss ValidWriteIdList"); } - String partitionPath = partition.getPath(); - //hdfs://xxxxx/user/hive/warehouse/username/data_id=200103 + //partitionPath eg: hdfs://xxxxx/user/hive/warehouse/username/data_id=200103 List lsPartitionPath = FileSystemTransferUtil.globList(fileSystem, partitionPath + "/*", false); @@ -412,7 +426,7 @@ && isValidBase(fileSystem, dirPath, base, validWriteIdList)) { } if (isFullAcid) { - fileCacheValue.setAcidInfo(new AcidInfo(partition.getPath(), deleteDeltas)); + fileCacheValue.setAcidInfo(new AcidInfo(partitionPath, deleteDeltas)); } else if (!deleteDeltas.isEmpty()) { throw new RuntimeException("No Hive Full Acid Table have delete_delta_* Dir."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index e1311237a603d5..9e2d1ad22e2e80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -464,6 +464,28 @@ public Optional> getSortedPartitionRanges(CatalogR return hivePartitionValues.getSortedPartitionRanges(); } + @Override + public SelectedPartitions initSelectedPartitions(Optional snapshot) { + // For Hive, read the cached HivePartitionValues once and freeze both the partition + // map and the cached sortedPartitionRanges from the same snapshot. This reuses the + // cached sorted ranges (no just-in-time rebuild) and keeps the two views consistent, + // avoiding the TOCTOU divergence where sortedPartitionRanges was re-read from cache + // at pruning time while the partition map was frozen earlier. + if (getDlaType() != DLAType.HIVE) { + return super.initSelectedPartitions(snapshot); + } + if (CollectionUtils.isEmpty(this.getPartitionColumns())) { + return SelectedPartitions.NOT_PRUNED; + } + HiveExternalMetaCache.HivePartitionValues hivePartitionValues = getHivePartitionValues( + MvccUtil.getSnapshotFromContext(this)); + Map nameToPartitionItems = hivePartitionValues.getNameToPartitionItem(); + Optional> sortedPartitionRanges + = hivePartitionValues.getSortedPartitionRanges(); + return new SelectedPartitions(nameToPartitionItems.size(), nameToPartitionItems, false, false, + sortedPartitionRanges); + } + public SelectedPartitions initHudiSelectedPartitions(Optional tableSnapshot) { if (getDlaType() != DLAType.HUDI) { return SelectedPartitions.NOT_PRUNED; @@ -482,6 +504,8 @@ public SelectedPartitions initHudiSelectedPartitions(Optional tab nameToPartitionItems.put(idToNameMap.get(entry.getKey()), entry.getValue()); } + // Hudi has no cached sorted ranges; leave it empty here and let PruneFileScanPartition + // build it lazily from this frozen map only when binary search filtering is enabled. return new SelectedPartitions(nameToPartitionItems.size(), nameToPartitionItems, false); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 54a6eb75b983ef..28e8b248620f79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -823,7 +823,8 @@ public List getFilesByTransaction(List partitions partition, txnValidIds, catalog.getCatalogProperty().getStoragePropertiesMap(), - isFullAcid))); + isFullAcid, + locationPath.getNormalizedLocation()))); } } } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index 3ea39987d021f7..cae6728c5f467f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -60,7 +60,6 @@ import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.TFileTextScanRangeParams; -import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.doris.thrift.TTransactionalHiveDeleteDeltaDesc; import org.apache.doris.thrift.TTransactionalHiveDesc; @@ -339,7 +338,7 @@ private void getFileSplitByPartitions(HiveExternalMetaCache cache, List defaultValue = IcebergUtils.parseIcebergLiteral(column.getDefaultValue(), dorisType); updateSchema.addColumn(column.getName(), dorisType, column.getComment(), defaultValue); } - private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, String columnName) { + private void applyPosition(UpdateSchema updateSchema, ColumnPosition position, ColumnPath columnPath, Schema schema, + String operation) throws UserException { + String columnName = columnPath.getFullPath(); if (position.isFirst()) { updateSchema.moveFirst(columnName); } else { - updateSchema.moveAfter(columnName, position.getLastCol()); + updateSchema.moveAfter(columnName, getPositionReferencePath(schema, columnPath, position, operation)); } } + private void validatePositionTarget(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + if (!columnPath.isNested()) { + return; + } + ResolvedColumnPath parentPath = resolveColumnPath(schema, columnPath.getParentPath(), operation); + if (!parentPath.getType().isStructType()) { + throw new UserException("Cannot apply column position to '" + columnPath.getFullPath() + + "': parent column path '" + parentPath.getFullPath() + "' is not a struct"); + } + } + + @VisibleForTesting + String getPositionReferencePath(ColumnPath columnPath, ColumnPosition position) { + if (position == null || position.isFirst() || !columnPath.isNested()) { + return position == null || position.isFirst() ? null : position.getLastCol(); + } + return columnPath.getParentPathString() + "." + position.getLastCol(); + } + + @VisibleForTesting + String getPositionReferencePath(Schema schema, ColumnPath columnPath, ColumnPosition position, String operation) + throws UserException { + if (position == null || position.isFirst()) { + return null; + } + ColumnPath referencePath = columnPath.isNested() + ? childPath(columnPath.getParentPath(), position.getLastCol()) + : ColumnPath.of(position.getLastCol()); + return resolveColumnPath(schema, referencePath, operation).getFullPath(); + } + private void refreshTable(ExternalTable dorisTable, long updateTime) { Optional> db = dorisCatalog.getDbForReplay(dorisTable.getRemoteDbName()); if (db.isPresent()) { @@ -839,12 +878,16 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { @Override public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { - validateCommonColumnInfo(column); + validateAddColumnMetadata(column, true); Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); + Schema schema = icebergTable.schema(); + validateNoCaseInsensitiveSiblingCollision( + schema.asStruct(), "", column.getName(), null, "add"); UpdateSchema updateSchema = icebergTable.updateSchema(); addOneColumn(updateSchema, column); if (position != null) { - applyPosition(updateSchema, position, column.getName()); + applyPosition(updateSchema, position, ColumnPath.of(column.getName()), schema, "add"); } try { executionAuthenticator.execute(() -> updateSchema.commit()); @@ -855,12 +898,55 @@ public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition po refreshTable(dorisTable, updateTime); } + @Override + public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, + long updateTime) throws UserException { + if (!columnPath.isNested()) { + addColumn(dorisTable, column, position, updateTime); + return; + } + validateNestedAddColumnMetadata(column, columnPath); + if (!column.isAllowNull()) { + throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); + } + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); + if (!parentPath.getType().isStructType()) { + throw new UserException("Parent column path '" + columnPath.getParentPathString() + + "' is not a struct in Iceberg table: " + icebergTable.name()); + } + validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), + parentPath.getColumnPath(), columnPath.getLeafName(), null, "add"); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + org.apache.iceberg.types.Type dorisType = + toIcebergTypeForSchemaChange(column.getType(), columnPath.getFullPath()); + updateSchema.addColumn(parentPath.getFullPath(), columnPath.getLeafName(), dorisType, + column.getComment()); + if (position != null) { + applyPosition(updateSchema, position, childPath(parentPath.getColumnPath(), columnPath.getLeafName()), + icebergTable.schema(), "add"); + } + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to add nested column: " + columnPath.getFullPath() + " to table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } + refreshTable(dorisTable, updateTime); + } + @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + for (Column column : columns) { + validateAddColumnMetadata(column, true); + validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); + } + validateNoCaseInsensitiveTopLevelCollisions(icebergTable.schema(), columns); + UpdateSchema updateSchema = icebergTable.updateSchema(); for (Column column : columns) { - validateCommonColumnInfo(column); addOneColumn(updateSchema, column); } try { @@ -875,8 +961,10 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + validateRowLineageColumnMutation(icebergTable, columnName, "drop"); + ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.deleteColumn(columnName); + updateSchema.deleteColumn(columnPath.getFullPath()); try { executionAuthenticator.execute(() -> updateSchema.commit()); } catch (Exception e) { @@ -886,12 +974,38 @@ public void dropColumn(ExternalTable dorisTable, String columnName, long updateT refreshTable(dorisTable, updateTime); } + @Override + public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long updateTime) throws UserException { + if (!columnPath.isNested()) { + dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); + return; + } + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + updateSchema.deleteColumn(resolvedPath.getFullPath()); + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to drop nested column: " + columnPath.getFullPath() + " from table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } + refreshTable(dorisTable, updateTime); + } + @Override public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + validateRowLineageColumnMutation(icebergTable, oldName, "rename"); + validateRowLineageColumnMutation(icebergTable, newName, "rename to"); + Schema schema = icebergTable.schema(); + ResolvedColumnPath oldPath = resolveColumnPath(schema, ColumnPath.of(oldName), "rename"); + validateNoCaseInsensitiveSiblingCollision( + schema.asStruct(), "", newName, oldPath.getField(), "rename"); UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.renameColumn(oldName, newName); + applyRenameColumn(schema, updateSchema, oldPath, newName); try { executionAuthenticator.execute(() -> updateSchema.commit()); } catch (Exception e) { @@ -901,42 +1015,112 @@ public void renameColumn(ExternalTable dorisTable, String oldName, String newNam refreshTable(dorisTable, updateTime); } + @Override + public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String newName, long updateTime) + throws UserException { + if (!columnPath.isNested()) { + renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); + return; + } + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "rename"); + ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "rename"); + validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), + parentPath.getColumnPath(), newName, resolvedPath.getField(), "rename"); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + applyRenameColumn(icebergTable.schema(), updateSchema, resolvedPath, newName); + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to rename nested column: " + columnPath.getFullPath() + " to " + newName + + " in table: " + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } + refreshTable(dorisTable, updateTime); + } + + private void applyRenameColumn(Schema schema, UpdateSchema updateSchema, + ResolvedColumnPath oldPath, String newName) { + String oldFullPath = oldPath.getFullPath(); + ColumnPath renamedPath = oldPath.getColumnPath().isNested() + ? childPath(oldPath.getColumnPath().getParentPath(), newName) + : ColumnPath.of(newName); + String renamedFullPath = renamedPath.getFullPath(); + boolean identifierFieldRenamed = false; + Set renamedIdentifierFields = new TreeSet<>(); + int renamedFieldId = oldPath.getField().fieldId(); + // Iceberg 1.10.1 does not preserve full identifier paths when an identifier field or one + // of its ancestors is renamed. Use field identity so dotted sibling names are not mistaken for descendants. + for (int identifierFieldId : schema.identifierFieldIds()) { + String identifierField = schema.findColumnName(identifierFieldId); + boolean isRenamedField = identifierFieldId == renamedFieldId; + boolean isDescendant = TypeUtil.ancestorFields(schema, identifierFieldId).stream() + .anyMatch(field -> field.fieldId() == renamedFieldId); + if (isRenamedField || isDescendant) { + renamedIdentifierFields.add(renamedFullPath + identifierField.substring(oldFullPath.length())); + identifierFieldRenamed = true; + } else { + renamedIdentifierFields.add(identifierField); + } + } + + updateSchema.renameColumn(oldFullPath, newName); + if (identifierFieldRenamed) { + updateSchema.setIdentifierFields(renamedIdentifierFields); + } + } + @Override public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { + // This overload predates nullableSpecified/commentSpecified. Keep its top-level values + // explicit while delegating to the path-aware implementation. + Column explicitColumn = new Column(column); + explicitColumn.setIsKey(false); + explicitColumn.setNullableSpecified(true); + explicitColumn.setCommentSpecified(true); + modifyColumn(dorisTable, ColumnPath.of(column.getName()), explicitColumn, position, updateTime); + } + + private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, + ColumnPosition position, long updateTime) throws UserException { Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); - NestedField currentCol = icebergTable.schema().findField(column.getName()); + validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); + NestedField currentCol = icebergTable.schema().asStruct() + .caseInsensitiveField(columnPath.getTopLevelName()); if (currentCol == null) { - throw new UserException("Column " + column.getName() + " does not exist"); + throw new UserException("Column " + columnPath.getTopLevelName() + " does not exist"); } + ResolvedColumnPath resolvedPath = new ResolvedColumnPath(ColumnPath.of(currentCol.name()), + currentCol.type(), currentCol); - validateCommonColumnInfo(column); - UpdateSchema updateSchema = icebergTable.updateSchema(); - + validateModifyColumnMetadata(column, resolvedPath.getFullPath(), true); + org.apache.iceberg.types.Type targetType; if (column.getType().isComplexType()) { - // Complex type processing branch validateForModifyComplexColumn(column, currentCol); - applyComplexTypeChange(updateSchema, column.getName(), currentCol.type(), column.getType()); - if (column.isAllowNull()) { - updateSchema.makeColumnOptional(column.getName()); - } - if (!Objects.equals(currentCol.doc(), column.getComment())) { - updateSchema.updateColumnDoc(column.getName(), column.getComment()); - } + targetType = currentCol.type(); } else { - // Primitive type processing (existing logic) validateForModifyColumn(column, currentCol); - Type icebergType = IcebergUtils.dorisTypeToIcebergType(column.getType()); - updateSchema.updateColumn(column.getName(), icebergType.asPrimitiveType(), column.getComment()); - if (column.isAllowNull()) { - // we can change a required column to optional, but not the other way around - // because we don't know whether there is existing data with null values. - updateSchema.makeColumnOptional(column.getName()); + targetType = resolvePrimitiveTypeForModify( + currentCol.type(), column.getType(), resolvedPath.getFullPath()); + } + + UpdateSchema updateSchema = icebergTable.updateSchema(); + String targetComment = resolveTargetComment(currentCol, column); + if (column.getType().isComplexType()) { + applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), + column.getType()); + if (!Objects.equals(currentCol.doc(), targetComment)) { + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); } + } else { + applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, + targetType.asPrimitiveType(), targetComment); } + applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); if (position != null) { - applyPosition(updateSchema, position, column.getName()); + applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); } try { executionAuthenticator.execute(() -> updateSchema.commit()); @@ -947,18 +1131,158 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition refreshTable(dorisTable, updateTime); } + @Override + public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, + long updateTime) throws UserException { + if (!columnPath.isNested()) { + modifyTopLevelColumn(dorisTable, columnPath, column, position, updateTime); + return; + } + + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); + NestedField currentCol = resolvedPath.getField(); + validateCollectionPseudoFieldComment( + icebergTable.schema(), resolvedPath, column.getComment(), column.isCommentSpecified()); + if (position != null) { + validatePositionTarget(icebergTable.schema(), resolvedPath.getColumnPath(), "modify"); + } + + validateNestedModifyColumnMetadata(column, resolvedPath.getFullPath()); + org.apache.iceberg.types.Type targetType; + if (column.getType().isComplexType()) { + validateForModifyComplexColumn(column, currentCol, columnPath.getFullPath()); + targetType = currentCol.type(); + } else { + validateForModifyColumn(column, currentCol, columnPath.getFullPath()); + targetType = resolvePrimitiveTypeForModify( + currentCol.type(), column.getType(), resolvedPath.getFullPath()); + } + + UpdateSchema updateSchema = icebergTable.updateSchema(); + String targetComment = resolveTargetComment(currentCol, column); + if (column.getType().isComplexType()) { + applyComplexTypeChange(updateSchema, resolvedPath.getFullPath(), currentCol.type(), + column.getType()); + if (!Objects.equals(currentCol.doc(), targetComment)) { + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), targetComment); + } + } else { + applyPrimitiveColumnChange(updateSchema, resolvedPath.getFullPath(), currentCol, + targetType.asPrimitiveType(), targetComment); + } + applyExplicitNullableChange(updateSchema, resolvedPath.getFullPath(), column); + + if (position != null) { + applyPosition(updateSchema, position, resolvedPath.getColumnPath(), icebergTable.schema(), "modify"); + } + + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to modify nested column: " + columnPath.getFullPath() + " in table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } + refreshTable(dorisTable, updateTime); + } + + @Override + public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) + throws UserException { + Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + if (!columnPath.isNested()) { + validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); + } + ResolvedColumnPath resolvedPath = resolveColumnPath( + icebergTable.schema(), columnPath, "modify comment"); + validateCollectionPseudoFieldComment(icebergTable.schema(), resolvedPath, comment, true); + + UpdateSchema updateSchema = icebergTable.updateSchema(); + updateSchema.updateColumnDoc(resolvedPath.getFullPath(), StringUtils.defaultString(comment)); + try { + executionAuthenticator.execute(() -> updateSchema.commit()); + } catch (Exception e) { + throw new UserException("Failed to modify column comment: " + columnPath.getFullPath() + " in table: " + + icebergTable.name() + ", error message is: " + e.getMessage(), e); + } + refreshTable(dorisTable, updateTime); + } + + private void validateCollectionPseudoFieldComment(Schema schema, ResolvedColumnPath resolvedPath, + String comment, boolean commentSpecified) throws UserException { + if (!resolvedPath.getColumnPath().isNested() + || (!commentSpecified && StringUtils.isEmpty(comment))) { + return; + } + ResolvedColumnPath parentPath = resolveColumnPath( + schema, resolvedPath.getColumnPath().getParentPath(), "modify comment"); + if (parentPath.getType().isListType() || parentPath.getType().isMapType()) { + throw new UserException("Iceberg does not support comments on collection element or value fields: " + + resolvedPath.getFullPath()); + } + } + + private void applyExplicitNullableChange(UpdateSchema updateSchema, String columnPath, Column column) { + if (column.isNullableSpecified() && column.isAllowNull()) { + updateSchema.makeColumnOptional(columnPath); + } + } + + private String resolveTargetComment(NestedField currentCol, Column column) { + return column.isCommentSpecified() ? column.getComment() : currentCol.doc(); + } + + private org.apache.iceberg.types.Type.PrimitiveType resolvePrimitiveTypeForModify( + org.apache.iceberg.types.Type currentIcebergType, + org.apache.doris.catalog.Type requestedDorisType, String columnPath) throws UserException { + if (isSameMappedDorisType(mappedDorisType(currentIcebergType), requestedDorisType)) { + return currentIcebergType.asPrimitiveType(); + } + org.apache.iceberg.types.Type.PrimitiveType currentType = currentIcebergType.asPrimitiveType(); + org.apache.iceberg.types.Type.PrimitiveType targetType = + toIcebergTypeForSchemaChange(requestedDorisType, columnPath).asPrimitiveType(); + if (!currentType.equals(targetType) && !TypeUtil.isPromotionAllowed(currentType, targetType)) { + throw new UserException("Cannot change column type: " + columnPath + ": " + + currentType + " -> " + targetType); + } + return targetType; + } + + private void applyPrimitiveColumnChange(UpdateSchema updateSchema, String columnPath, + NestedField currentCol, org.apache.iceberg.types.Type.PrimitiveType targetType, + String targetComment) { + if (!currentCol.type().equals(targetType)) { + updateSchema.updateColumn(columnPath, targetType, targetComment); + } else if (!Objects.equals(currentCol.doc(), targetComment)) { + updateSchema.updateColumnDoc(columnPath, targetComment); + } + } + private void validateForModifyColumn(Column column, NestedField currentCol) throws UserException { + validateForModifyColumn(column, currentCol, column.getName()); + } + + private void validateForModifyColumn(Column column, NestedField currentCol, String columnPath) + throws UserException { // check complex type if (column.getType().isComplexType()) { throw new UserException("Modify column type to non-primitive type is not supported: " + column.getType()); } + if (!currentCol.type().isPrimitiveType()) { + throw new UserException("Modify column type from complex to primitive is not supported: " + columnPath); + } // check nullable if (currentCol.isOptional() && !column.isAllowNull()) { - throw new UserException("Can not change nullable column " + column.getName() + " to not null"); + throw new UserException("Can not change nullable column " + columnPath + " to not null"); } } private void validateForModifyComplexColumn(Column column, NestedField currentCol) throws UserException { + validateForModifyComplexColumn(column, currentCol, column.getName()); + } + + private void validateForModifyComplexColumn(Column column, NestedField currentCol, String columnPath) + throws UserException { if (!column.getType().isComplexType()) { throw new UserException("Modify column type to non-complex type is not supported: " + column.getType()); } @@ -968,7 +1292,7 @@ private void validateForModifyComplexColumn(Column column, NestedField currentCo + column.getName()); } - org.apache.doris.catalog.Type oldDorisType = IcebergUtils.icebergTypeToDorisType(oldIcebergType, false, false); + org.apache.doris.catalog.Type oldDorisType = mappedDorisType(oldIcebergType); org.apache.doris.catalog.Type newDorisType = column.getType(); if (!isSameComplexCategory(oldIcebergType, newDorisType)) { throw new UserException("Cannot change complex column type category from " @@ -979,14 +1303,169 @@ private void validateForModifyComplexColumn(Column column, NestedField currentCo } catch (DdlException e) { throw new UserException(e.getMessage(), e); } + validateComplexTypeChanges(oldIcebergType, newDorisType, columnPath); if (currentCol.isOptional() && !column.isAllowNull()) { - throw new UserException("Cannot change nullable column " + column.getName() + " to not null"); + throw new UserException("Cannot change nullable column " + columnPath + " to not null"); } if (column.getDefaultValue() != null || column.getDefaultValueExprDef() != null) { throw new UserException("Complex type default value only supports NULL"); } } + @VisibleForTesting + org.apache.iceberg.types.Type resolveNestedColumnPath(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + return resolveColumnPath(schema, columnPath, operation).getType(); + } + + @VisibleForTesting + String getCanonicalColumnPath(Schema schema, ColumnPath columnPath, String operation) throws UserException { + return resolveColumnPath(schema, columnPath, operation).getFullPath(); + } + + private ResolvedColumnPath resolveColumnPath(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + org.apache.iceberg.types.Type currentType = schema.asStruct(); + NestedField currentField = null; + String currentPath = ""; + List canonicalParts = new ArrayList<>(); + for (String part : columnPath.getParts()) { + if (!currentPath.isEmpty()) { + currentPath += "."; + } + currentPath += part; + + if (currentType.isStructType()) { + NestedField field = currentType.asStructType().caseInsensitiveField(part); + if (field == null) { + throw new UserException("Column path does not exist in Iceberg schema: " + + columnPath.getFullPath()); + } + canonicalParts.add(field.name()); + currentField = field; + currentType = field.type(); + } else if (currentType.isListType()) { + Types.ListType listType = currentType.asListType(); + NestedField elementField = listType.field(listType.elementId()); + if (!elementField.name().equalsIgnoreCase(part)) { + throw new UserException("Expected array element path at '" + currentPath + + "' for Iceberg column path: " + columnPath.getFullPath()); + } + canonicalParts.add(elementField.name()); + currentField = elementField; + currentType = listType.elementType(); + } else if (currentType.isMapType()) { + Types.MapType mapType = currentType.asMapType(); + NestedField keyField = mapType.field(mapType.keyId()); + if (keyField.name().equalsIgnoreCase(part)) { + throw new UserException("Cannot " + operation + " MAP key nested column: " + + columnPath.getFullPath()); + } + NestedField valueField = mapType.field(mapType.valueId()); + if (!valueField.name().equalsIgnoreCase(part)) { + throw new UserException("Expected map value path at '" + currentPath + + "' for Iceberg column path: " + columnPath.getFullPath()); + } + canonicalParts.add(valueField.name()); + currentField = valueField; + currentType = mapType.valueType(); + } else { + throw new UserException("Cannot resolve nested field under primitive column path: " + + columnPath.getFullPath()); + } + } + return new ResolvedColumnPath(ColumnPath.of(canonicalParts), currentType, currentField); + } + + @VisibleForTesting + org.apache.iceberg.types.Type validateNestedStructField(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + return validateNestedStructFieldPath(schema, columnPath, operation).getType(); + } + + private ResolvedColumnPath validateNestedStructFieldPath(Schema schema, ColumnPath columnPath, String operation) + throws UserException { + ResolvedColumnPath parentPath = resolveColumnPath(schema, columnPath.getParentPath(), operation); + org.apache.iceberg.types.Type parentType = parentPath.getType(); + if (!parentType.isStructType()) { + throw new UserException("Parent column path '" + columnPath.getParentPathString() + + "' is not a struct for Iceberg nested " + operation + ": " + columnPath.getFullPath()); + } + NestedField field = parentType.asStructType().caseInsensitiveField(columnPath.getLeafName()); + if (field == null) { + throw new UserException("Column path does not exist in Iceberg schema: " + columnPath.getFullPath()); + } + return new ResolvedColumnPath(childPath(parentPath.getColumnPath(), field.name()), field.type(), field); + } + + private ColumnPath childPath(ColumnPath parentPath, String childName) { + List parts = new ArrayList<>(parentPath.getParts()); + parts.add(childName); + return ColumnPath.of(parts); + } + + @VisibleForTesting + void validateNoCaseInsensitiveSiblingCollision(Types.StructType parentType, ColumnPath parentPath, + String targetName, NestedField sourceField, String operation) throws UserException { + validateNoCaseInsensitiveSiblingCollision( + parentType, parentPath.getFullPath(), targetName, sourceField, operation); + } + + private void validateNoCaseInsensitiveSiblingCollision(Types.StructType parentType, String parentPath, + String targetName, NestedField sourceField, String operation) throws UserException { + NestedField conflictingField = parentType.caseInsensitiveField(targetName); + if (conflictingField != null + && (sourceField == null || conflictingField.fieldId() != sourceField.fieldId())) { + String targetPath = parentPath.isEmpty() ? targetName : parentPath + "." + targetName; + String conflictingPath = parentPath.isEmpty() + ? conflictingField.name() : parentPath + "." + conflictingField.name(); + String columnDescription = parentPath.isEmpty() ? "column" : "nested column"; + throw new UserException("Cannot " + operation + " " + columnDescription + " '" + targetPath + + "': conflicts with existing Iceberg field '" + conflictingPath + "' (case-insensitive)"); + } + } + + private void validateNoCaseInsensitiveTopLevelCollisions(Schema schema, List columns) + throws UserException { + Set requestedNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (Column column : columns) { + validateNoCaseInsensitiveSiblingCollision( + schema.asStruct(), "", column.getName(), null, "add"); + if (!requestedNames.add(column.getName())) { + throw new UserException("Cannot add column '" + column.getName() + + "': conflicts with another requested column (case-insensitive)"); + } + } + } + + private static class ResolvedColumnPath { + private final ColumnPath columnPath; + private final org.apache.iceberg.types.Type type; + private final NestedField field; + + private ResolvedColumnPath(ColumnPath columnPath, org.apache.iceberg.types.Type type, NestedField field) { + this.columnPath = columnPath; + this.type = type; + this.field = field; + } + + private ColumnPath getColumnPath() { + return columnPath; + } + + private String getFullPath() { + return columnPath.getFullPath(); + } + + private org.apache.iceberg.types.Type getType() { + return type; + } + + private NestedField getField() { + return field; + } + } + private boolean isSameComplexCategory(Type oldIcebergType, org.apache.doris.catalog.Type newDorisType) { switch (oldIcebergType.typeId()) { case STRUCT: @@ -1000,6 +1479,59 @@ private boolean isSameComplexCategory(Type oldIcebergType, org.apache.doris.cata } } + private void validateComplexTypeChanges(org.apache.iceberg.types.Type oldIcebergType, + org.apache.doris.catalog.Type newDorisType, String path) throws UserException { + switch (oldIcebergType.typeId()) { + case STRUCT: + validateStructTypeChanges(oldIcebergType.asStructType(), (StructType) newDorisType, path); + break; + case LIST: + Types.ListType oldListType = oldIcebergType.asListType(); + validateExistingTypeChange(oldListType.elementType(), ((ArrayType) newDorisType).getItemType(), + path + "." + oldListType.field(oldListType.elementId()).name()); + break; + case MAP: + Types.MapType oldMapType = oldIcebergType.asMapType(); + MapType newMapType = (MapType) newDorisType; + org.apache.doris.catalog.Type oldDorisKeyType = mappedDorisType(oldMapType.keyType()); + if (!isSameMappedDorisType(oldDorisKeyType, newMapType.getKeyType())) { + throw new UserException("Cannot change MAP key type from " + + oldDorisKeyType.toSql() + " to " + newMapType.getKeyType().toSql()); + } + validateExistingTypeChange(oldMapType.valueType(), newMapType.getValueType(), + path + "." + oldMapType.field(oldMapType.valueId()).name()); + break; + default: + throw new UserException("Unsupported complex type for modify: " + oldIcebergType); + } + } + + private void validateStructTypeChanges(Types.StructType oldStructType, + StructType newStructType, String path) throws UserException { + List oldFields = oldStructType.fields(); + List newFields = newStructType.getFields(); + for (int i = 0; i < oldFields.size(); i++) { + NestedField oldField = oldFields.get(i); + validateExistingTypeChange(oldField.type(), newFields.get(i).getType(), + path + "." + oldField.name()); + } + for (int i = oldFields.size(); i < newFields.size(); i++) { + StructField newField = newFields.get(i); + toIcebergTypeForSchemaChange(newField.getType(), path + "." + newField.getName()); + } + } + + private void validateExistingTypeChange(org.apache.iceberg.types.Type oldIcebergType, + org.apache.doris.catalog.Type newDorisType, String path) throws UserException { + if (oldIcebergType.isPrimitiveType()) { + if (!isSameMappedDorisType(mappedDorisType(oldIcebergType), newDorisType)) { + toIcebergTypeForSchemaChange(newDorisType, path); + } + return; + } + validateComplexTypeChanges(oldIcebergType, newDorisType, path); + } + private void applyComplexTypeChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldIcebergType, org.apache.doris.catalog.Type newDorisType) throws UserException { @@ -1019,7 +1551,8 @@ private void applyComplexTypeChange(UpdateSchema updateSchema, String path, } private void applyStructChange(UpdateSchema updateSchema, String path, - Types.StructType oldStructType, StructType newStructType) throws UserException { + Types.StructType oldStructType, StructType newStructType) + throws UserException { List oldFields = oldStructType.fields(); List newFields = newStructType.getFields(); @@ -1034,27 +1567,26 @@ private void applyStructChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldFieldType = oldField.type(); org.apache.doris.catalog.Type newFieldType = newField.getType(); + String targetComment = newField.isCommentSpecified() + ? newField.getComment() : oldField.doc(); if (oldFieldType.isPrimitiveType()) { - org.apache.doris.catalog.Type oldDorisFieldType = - IcebergUtils.icebergTypeToDorisType(oldFieldType, false, false); - boolean typeChanged = !oldDorisFieldType.equals(newFieldType); - boolean commentChanged = !Objects.equals(oldField.doc(), newField.getComment()); - if (typeChanged || commentChanged) { + org.apache.doris.catalog.Type oldDorisFieldType = mappedDorisType(oldFieldType); + boolean typeChanged = !isSameMappedDorisType(oldDorisFieldType, newFieldType); + boolean commentChanged = !Objects.equals(oldField.doc(), targetComment); + if (typeChanged) { org.apache.iceberg.types.Type newIcebergFieldType = - IcebergUtils.dorisTypeToIcebergType(newFieldType); + toIcebergTypeForSchemaChange(newFieldType, fieldPath); updateSchema.updateColumn(fieldPath, newIcebergFieldType.asPrimitiveType(), - newField.getComment()); + targetComment); + } else if (commentChanged) { + updateSchema.updateColumnDoc(fieldPath, targetComment); } } else { applyComplexTypeChange(updateSchema, fieldPath, oldFieldType, newFieldType); - if (!Objects.equals(oldField.doc(), newField.getComment())) { - updateSchema.updateColumnDoc(fieldPath, newField.getComment()); + if (!Objects.equals(oldField.doc(), targetComment)) { + updateSchema.updateColumnDoc(fieldPath, targetComment); } } - - if (!oldField.isOptional() && newField.getContainsNull()) { - updateSchema.makeColumnOptional(fieldPath); - } } for (int i = oldFields.size(); i < newFields.size(); i++) { @@ -1063,13 +1595,14 @@ private void applyStructChange(UpdateSchema updateSchema, String path, throw new UserException("New struct field '" + newField.getName() + "' must be nullable"); } org.apache.iceberg.types.Type newFieldIcebergType = - IcebergUtils.dorisTypeToIcebergType(newField.getType()); + toIcebergTypeForSchemaChange(newField.getType(), path + "." + newField.getName()); updateSchema.addColumn(path, newField.getName(), newFieldIcebergType, newField.getComment()); } } private void applyListChange(UpdateSchema updateSchema, String path, - Types.ListType oldListType, ArrayType newArrayType) throws UserException { + Types.ListType oldListType, ArrayType newArrayType) + throws UserException { String elementPath = path + "." + oldListType.field(oldListType.elementId()).name(); if (oldListType.isElementOptional() && !newArrayType.getContainsNull()) { throw new UserException("Cannot change nullable column " + elementPath + " to not null"); @@ -1077,28 +1610,24 @@ private void applyListChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldElementType = oldListType.elementType(); org.apache.doris.catalog.Type newElementType = newArrayType.getItemType(); if (oldElementType.isPrimitiveType()) { - org.apache.doris.catalog.Type oldDorisElementType = - IcebergUtils.icebergTypeToDorisType(oldElementType, false, false); - if (!oldDorisElementType.equals(newElementType)) { + org.apache.doris.catalog.Type oldDorisElementType = mappedDorisType(oldElementType); + if (!isSameMappedDorisType(oldDorisElementType, newElementType)) { org.apache.iceberg.types.Type newIcebergElementType = - IcebergUtils.dorisTypeToIcebergType(newElementType); + toIcebergTypeForSchemaChange(newElementType, elementPath); updateSchema.updateColumn(elementPath, newIcebergElementType.asPrimitiveType(), null); } } else { applyComplexTypeChange(updateSchema, elementPath, oldElementType, newElementType); } - if (!oldListType.isElementOptional() && newArrayType.getContainsNull()) { - updateSchema.makeColumnOptional(elementPath); - } } private void applyMapChange(UpdateSchema updateSchema, String path, - Types.MapType oldMapType, MapType newMapType) throws UserException { + Types.MapType oldMapType, MapType newMapType) + throws UserException { org.apache.iceberg.types.Type oldKeyType = oldMapType.keyType(); org.apache.doris.catalog.Type newKeyType = newMapType.getKeyType(); - org.apache.doris.catalog.Type oldDorisKeyType = - IcebergUtils.icebergTypeToDorisType(oldKeyType, false, false); - if (!oldDorisKeyType.equals(newKeyType)) { + org.apache.doris.catalog.Type oldDorisKeyType = mappedDorisType(oldKeyType); + if (!isSameMappedDorisType(oldDorisKeyType, newKeyType)) { throw new UserException("Cannot change MAP key type from " + oldDorisKeyType.toSql() + " to " + newKeyType.toSql()); } @@ -1110,22 +1639,29 @@ private void applyMapChange(UpdateSchema updateSchema, String path, org.apache.iceberg.types.Type oldValueType = oldMapType.valueType(); org.apache.doris.catalog.Type newValueType = newMapType.getValueType(); if (oldValueType.isPrimitiveType()) { - org.apache.doris.catalog.Type oldDorisValueType = - IcebergUtils.icebergTypeToDorisType(oldValueType, false, false); - if (!oldDorisValueType.equals(newValueType)) { + org.apache.doris.catalog.Type oldDorisValueType = mappedDorisType(oldValueType); + if (!isSameMappedDorisType(oldDorisValueType, newValueType)) { org.apache.iceberg.types.Type newIcebergValueType = - IcebergUtils.dorisTypeToIcebergType(newValueType); + toIcebergTypeForSchemaChange(newValueType, valuePath); updateSchema.updateColumn(valuePath, newIcebergValueType.asPrimitiveType(), null); } } else { applyComplexTypeChange(updateSchema, valuePath, oldValueType, newValueType); } - if (!oldMapType.isValueOptional() && newMapType.getIsValueContainsNull()) { - updateSchema.makeColumnOptional(valuePath); - } } - private void validateCommonColumnInfo(Column column) throws UserException { + private void validateCommonColumnInfo(Column column, boolean rejectKey) throws UserException { + validateCommonColumnMetadata(column, rejectKey); + toIcebergTypeForSchemaChange(column.getType(), column.getName()); + } + + private void validateCommonColumnMetadata(Column column, boolean rejectKey) throws UserException { + if (rejectKey && column.isKey()) { + throw new UserException("KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + } + if (column.isGeneratedColumn()) { + throw new UserException("Generated columns are not supported for Iceberg ADD/MODIFY COLUMN"); + } // check aggregation method if (column.isAggregated()) { throw new UserException("Can not specify aggregation method for iceberg table column"); @@ -1136,16 +1672,87 @@ private void validateCommonColumnInfo(Column column) throws UserException { } } + private org.apache.doris.catalog.Type mappedDorisType(org.apache.iceberg.types.Type icebergType) { + return IcebergUtils.icebergTypeToDorisType(icebergType, + dorisCatalog.getEnableMappingVarbinary(), dorisCatalog.getEnableMappingTimestampTz()); + } + + private boolean isSameMappedDorisType(org.apache.doris.catalog.Type mappedType, + org.apache.doris.catalog.Type requestedType) { + // ScalarType.equals does not compare VARBINARY length, but Iceberg FIXED uses the + // mapped length to distinguish an unchanged view from a requested type change. + return mappedType.equals(requestedType) + && (!mappedType.isVarbinaryType() || mappedType.getLength() == requestedType.getLength()); + } + + private org.apache.iceberg.types.Type toIcebergTypeForSchemaChange( + org.apache.doris.catalog.Type dorisType, String columnPath) throws UserException { + try { + return IcebergUtils.dorisTypeToIcebergType(dorisType); + } catch (UnsupportedOperationException | IllegalArgumentException e) { + throw new UserException("Type " + dorisType.toSql() + + " is not supported for Iceberg column " + columnPath, e); + } + } + + private void validateNestedAddColumnMetadata(Column column, ColumnPath columnPath) throws UserException { + validateCommonColumnInfo(column, true); + if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { + throw new UserException("DEFAULT and ON UPDATE are not supported for Iceberg nested ADD COLUMN: " + + columnPath.getFullPath()); + } + } + + private void validateNestedModifyColumnMetadata(Column column, String columnPath) throws UserException { + validateModifyColumnMetadata(column, columnPath, true); + } + + private void validateAddColumnMetadata(Column column, boolean rejectKey) throws UserException { + validateCommonColumnInfo(column, rejectKey); + if (column.hasOnUpdateDefaultValue()) { + throw new UserException("ON UPDATE is not supported for Iceberg ADD COLUMN: " + column.getName()); + } + } + + private void validateModifyColumnMetadata(Column column, String columnPath, boolean rejectKey) + throws UserException { + validateCommonColumnMetadata(column, rejectKey); + if (column.hasDefaultValue() || column.hasOnUpdateDefaultValue()) { + throw new UserException("Modifying default values is not supported for Iceberg columns: " + columnPath); + } + } + + private void validateRowLineageColumnMutation(Table icebergTable, String columnName, String operation) + throws UserException { + int formatVersion = IcebergUtils.getFormatVersion(icebergTable); + if (formatVersion >= IcebergUtils.ICEBERG_ROW_LINEAGE_MIN_VERSION + && IcebergUtils.isIcebergRowLineageColumn(columnName)) { + throw new UserException("Cannot " + operation + " Iceberg v" + formatVersion + + " reserved row lineage column: " + columnName); + } + } + @Override public void reorderColumns(ExternalTable dorisTable, List newOrder, long updateTime) throws UserException { if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + List canonicalOrder = new ArrayList<>(newOrder.size()); + Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (String columnName : newOrder) { + validateRowLineageColumnMutation(icebergTable, columnName, "reorder"); + String canonicalName = resolveColumnPath( + icebergTable.schema(), ColumnPath.of(columnName), "reorder").getFullPath(); + if (!canonicalNames.add(canonicalName)) { + throw new UserException("Duplicate column in reorder columns: " + columnName); + } + canonicalOrder.add(canonicalName); + } UpdateSchema updateSchema = icebergTable.updateSchema(); - updateSchema.moveFirst(newOrder.get(0)); - for (int i = 1; i < newOrder.size(); i++) { - updateSchema.moveAfter(newOrder.get(i), newOrder.get(i - 1)); + updateSchema.moveFirst(canonicalOrder.get(0)); + for (int i = 1; i < canonicalOrder.size(); i++) { + updateSchema.moveAfter(canonicalOrder.get(i), canonicalOrder.get(i - 1)); } try { executionAuthenticator.execute(() -> updateSchema.commit()); @@ -1383,7 +1990,7 @@ private Optional viewCatalog(SessionContext ctx) { return defaultViewCatalog; } - private Optional resolveDefaultViewCatalog(Catalog catalog, RESTSessionCatalog restSessionCatalog, + private Optional resolveDefaultViewCatalog(Catalog catalog, BaseViewSessionCatalog restSessionCatalog, boolean viewEnabled) { // Branch on whether this is a REST (session-aware) catalog, not on whether restSessionCatalog happens to // be built: for REST the default Catalog (asCatalog) is not a ViewCatalog, so views must come from the diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java index c3d7a88bf012df..44f194a0bf7511 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java @@ -29,7 +29,7 @@ import org.apache.doris.datasource.property.metastore.MetastoreProperties; import com.google.common.collect.Lists; -import org.apache.iceberg.rest.RESTSessionCatalog; +import org.apache.iceberg.catalog.BaseViewSessionCatalog; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -226,7 +226,7 @@ public boolean useSessionCatalog(SessionContext ctx) { } @Override - public RESTSessionCatalog getRestSessionCatalog() { + public BaseViewSessionCatalog getRestSessionCatalog() { IcebergRestProperties props = restProperties(); return props == null ? null : props.getRestSessionCatalog(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 95c9a6f26cc5c5..a17ff87b1131c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,14 +17,30 @@ package org.apache.doris.datasource.iceberg; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; + private final Optional>> nameMapping; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { + this(partitionInfo, snapshot, Optional.empty()); + } + + public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; + this.nameMapping = nameMapping.map(mapping -> { + Map> copy = new HashMap<>(); + mapping.forEach((id, names) -> copy.put(id, List.copyOf(names))); + return Map.copyOf(copy); + }); } public IcebergPartitionInfo getPartitionInfo() { @@ -34,4 +50,8 @@ public IcebergPartitionInfo getPartitionInfo() { public IcebergSnapshot getSnapshot() { return snapshot; } + + public Optional>> getNameMapping() { + return nameMapping; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java index aeb539d1f3fa3d..14047dd44ff26b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java @@ -80,6 +80,10 @@ public String getSysTableType() { return sysTableType; } + public boolean isPositionDeletesTable() { + return MetadataTableType.POSITION_DELETES.name().equalsIgnoreCase(sysTableType); + } + public Table getSysIcebergTable() { if (sysIcebergTable == null) { synchronized (this) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUserSessionCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUserSessionCatalog.java index 4aae96ead4a29e..e0d8b1c5b39d28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUserSessionCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUserSessionCatalog.java @@ -20,7 +20,7 @@ import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.property.metastore.IcebergRestProperties; -import org.apache.iceberg.rest.RESTSessionCatalog; +import org.apache.iceberg.catalog.BaseViewSessionCatalog; /** * Capability interface for an Iceberg catalog that supports per-user dynamic session @@ -47,7 +47,7 @@ public interface IcebergUserSessionCatalog { boolean useSessionCatalog(SessionContext ctx); /** The session-aware Iceberg REST catalog backing this catalog (may be null before initialization). */ - RESTSessionCatalog getRestSessionCatalog(); + BaseViewSessionCatalog getRestSessionCatalog(); /** The delegated-token mode used when attaching the user's credential to session requests. */ IcebergRestProperties.DelegatedTokenMode getDelegatedTokenMode(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 12b5dce11395fa..bf8f0980c3e5b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -79,14 +79,17 @@ import com.google.gson.reflect.TypeToken; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetadataTableUtils; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.MetricsModes; +import org.apache.iceberg.MetricsUtil; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; @@ -108,6 +111,10 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mapping.MappedField; +import org.apache.iceberg.mapping.MappedFields; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; @@ -1115,8 +1122,18 @@ private static long parseTimestampToMicros(String valueStr, TimestampType timest return epochSecond * 1_000_000L + microSecond; } - private static void updateIcebergColumnUniqueId(Column column, Types.NestedField icebergField) { + private static void updateIcebergColumnMetadata(Column column, Types.NestedField icebergField, + boolean enableMappingTimestampTz) { column.setUniqueId(icebergField.fieldId()); + if (icebergField.initialDefault() != null) { + String serializedDefault = serializeInitialDefault( + icebergField.type(), icebergField.initialDefault(), enableMappingTimestampTz); + // Column constructs complex children without Iceberg field metadata. Copy through the + // public default-info API so recursive fields retain their logical pre-add value. + Column defaultCarrier = new Column(column.getName(), column.getType(), false, null, + column.isAllowNull(), serializedDefault, ""); + column.setDefaultValueInfo(defaultCarrier); + } List icebergFields = Lists.newArrayList(); switch (icebergField.type().typeId()) { case LIST: @@ -1135,7 +1152,8 @@ private static void updateIcebergColumnUniqueId(Column column, Types.NestedField if (column.getChildren() != null) { List childColumns = column.getChildren(); for (int idx = 0; idx < childColumns.size(); idx++) { - updateIcebergColumnUniqueId(childColumns.get(idx), icebergFields.get(idx)); + updateIcebergColumnMetadata( + childColumns.get(idx), icebergFields.get(idx), enableMappingTimestampTz); } } } @@ -1192,7 +1210,7 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), true, null, true, initialDefault, field.doc(), true, -1); - updateIcebergColumnUniqueId(column, field); + updateIcebergColumnMetadata(column, field, enableMappingTimestampTz); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); if (timestampType.shouldAdjustToUTC()) { @@ -1293,7 +1311,7 @@ public static long getIcebergRowCount(ExternalTable tbl) { public static FileFormat getFileFormat(Table icebergTable) { Map properties = icebergTable.properties(); - String fileFormatName = resolveFileFormatName(icebergTable, properties); + String fileFormatName = resolveFileFormatName(properties); FileFormat fileFormat; if (fileFormatName.toLowerCase().contains(ORC_NAME)) { fileFormat = FileFormat.ORC; @@ -1305,7 +1323,7 @@ public static FileFormat getFileFormat(Table icebergTable) { return fileFormat; } - private static String resolveFileFormatName(Table icebergTable, Map properties) { + private static String resolveFileFormatName(Map properties) { // 1. Check "write-format" (nickname in Flink and Spark) if (properties.containsKey(WRITE_FORMAT)) { return properties.get(WRITE_FORMAT); @@ -1314,27 +1332,7 @@ private static String resolveFileFormatName(Table icebergTable, Map files = icebergTable.newScan().planFiles()) { - java.util.Iterator it = files.iterator(); - if (it.hasNext()) { - String format = it.next().file().format().name().toLowerCase(); - LOG.info("Iceberg table {} inferred file format {} from data files", icebergTable.name(), format); - return format; - } - } catch (Exception e) { - LOG.warn("Failed to infer file format from data files for table {}, defaulting to {}", - icebergTable.name(), PARQUET_NAME, e); - } + // Iceberg defaults the write format to Parquet when the table does not declare one. return PARQUET_NAME; } @@ -1839,7 +1837,8 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( } return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), - new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId())); + new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), + getNameMapping(icebergTable)); } return getLatestSnapshotCacheValue(dorisTable); } @@ -1920,12 +1919,51 @@ private static IcebergSnapshotCacheValue loadSnapshotCacheValue(ExternalTable do icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, icebergTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue(icebergPartitionInfo, latestIcebergSnapshot); + return new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, getNameMapping(icebergTable)); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } } + /** + * Extract the Iceberg name mapping while retaining the distinction between an absent property + * and a valid empty mapping. + */ + public static Optional>> getNameMapping(Table icebergTable) { + String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); + if (nameMappingJson == null || nameMappingJson.isEmpty()) { + return Optional.empty(); + } + try { + NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); + if (mapping == null) { + return Optional.empty(); + } + Map> result = new HashMap<>(); + extractMappingsFromNameMapping(mapping.asMappedFields(), result); + return Optional.of(result); + } catch (Exception e) { + LOG.warn("Failed to parse name mapping from Iceberg table properties", e); + return Optional.empty(); + } + } + + private static void extractMappingsFromNameMapping( + MappedFields mappingFields, Map> result) { + if (mappingFields == null) { + return; + } + for (MappedField mappedField : mappingFields.fields()) { + // Iceberg permits id-less wrapper entries; only their nested ID-bearing aliases can + // participate in Doris field-id lookup and in the immutable snapshot cache. + if (mappedField.id() != null) { + result.put(mappedField.id(), new ArrayList<>(mappedField.names())); + } + extractMappingsFromNameMapping(mappedField.nestedMapping(), result); + } + } + private static Table loadIcebergTableWithSession(ExternalTable dorisTable) { IcebergExternalCatalog catalog = (IcebergExternalCatalog) dorisTable.getCatalog(); IcebergMetadataOps ops = (IcebergMetadataOps) catalog.getMetadataOps(); @@ -1981,10 +2019,25 @@ public static Schema appendRowLineageFieldsForV3(Schema schema) { MetadataColumns.ROW_ID, MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER)); } + public static boolean shouldCollectColumnStats(Table table, Schema writerSchema) { + MetricsConfig metricsConfig = MetricsConfig.forTable(table); + if (getFileFormat(table) == FileFormat.ORC) { + // Match the footer collectors: ORC reports top-level collection counts, while Parquet reports leaf fields. + return writerSchema.columns().stream() + .anyMatch(field -> MetricsUtil.metricsMode(writerSchema, metricsConfig, field.fieldId()) + != MetricsModes.None.get()); + } + return TypeUtil.indexById(writerSchema.asStruct()).values().stream() + .filter(field -> field.type().isPrimitiveType()) + .anyMatch(field -> MetricsUtil.metricsMode(writerSchema, metricsConfig, field.fieldId()) + != MetricsModes.None.get()); + } + public static int getFormatVersion(Table table) { int formatVersion = 2; // default format version : 2 - if (table instanceof BaseTable) { - formatVersion = ((BaseTable) table).operations().current().formatVersion(); + if (table instanceof HasTableOperations) { + // TransactionTable exposes the real format version through operations, not table properties. + formatVersion = ((HasTableOperations) table).operations().current().formatVersion(); } else if (table != null && table.properties() != null) { String version = table.properties().get(TableProperties.FORMAT_VERSION); if (version != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/ReauthenticatingRestSessionCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/ReauthenticatingRestSessionCatalog.java new file mode 100644 index 00000000000000..c39b0a73381124 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/ReauthenticatingRestSessionCatalog.java @@ -0,0 +1,283 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.BaseViewSessionCatalog; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SessionCatalog.SessionContext; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.iceberg.rest.RESTSessionCatalog; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +/** + * A session catalog that transparently recovers from an expired/rejected credential on an Iceberg REST + * catalog, instead of failing every request until the FE restarts. + * + *

Why this exists. {@link RESTSessionCatalog} obtains an OAuth2 token at {@code initialize()} and + * keeps it refreshed in the background. If a refresh permanently fails (e.g. the auth server was briefly + * unreachable or overloaded at refresh time), the client is left holding a stale token and every subsequent + * request fails with {@link NotAuthorizedException} (HTTP 401) forever — the client has no re-authentication + * path of its own, and neither {@code REFRESH CATALOG} nor metadata-cache invalidation rebuilds it. The only + * recovery is building a fresh client, which is exactly what this wrapper does: on a 401 from a request made + * under the catalog's own identity, it rebuilds the delegate {@link RESTSessionCatalog} (forcing a fresh + * OAuth2 token fetch), closes the wedged one, and retries the operation once. If the retry also fails, the + * error propagates unchanged. + * + *

Why a wrapper at the session-catalog level. Everything Doris hands out for a REST catalog — the + * default {@code asCatalog(empty)} used by {@code IcebergMetadataOps}, the {@code asViewCatalog(empty)} view + * path, and per-user delegated sessions — is a thin view that calls back into this class (see + * {@link org.apache.iceberg.catalog.BaseSessionCatalog.AsCatalog}). Wrapping here means no holder of any of + * those references ever sees a stale client after recovery, and non-REST catalogs are untouched because only + * {@code IcebergRestProperties} constructs this class. + * + *

What is retried. Both reads and mutations: a 401 is rejected by the server before the request is + * processed, so retrying after re-authentication cannot double-apply an operation. Requests that carry a + * per-user delegated credential are not recovered — a 401 there means that user's token is invalid, + * and rebuilding the shared client cannot (and must not) fix it. + * + *

Boundary. Recovery triggers on catalog-level operations. Objects already handed out (a loaded + * {@link Table}/{@link View}, a builder from {@code buildTable}/{@code buildView}) keep their own reference + * to the client they were created with; if the credential expires between obtaining such an object and using + * it, that one use can still fail with a 401. The next catalog-level operation rebuilds the client, after + * which reloading the object succeeds. + */ +public class ReauthenticatingRestSessionCatalog extends BaseViewSessionCatalog implements Closeable { + + private static final Logger LOG = LogManager.getLogger(ReauthenticatingRestSessionCatalog.class); + + private final Supplier delegateBuilder; + private volatile RESTSessionCatalog delegate; + + public ReauthenticatingRestSessionCatalog(RESTSessionCatalog initialDelegate, + Supplier delegateBuilder) { + this.delegate = initialDelegate; + this.delegateBuilder = delegateBuilder; + } + + @VisibleForTesting + RESTSessionCatalog currentDelegate() { + return delegate; + } + + private T withAuthRecovery(SessionContext context, Supplier op) { + RESTSessionCatalog attemptedOn = delegate; + try { + return op.get(); + } catch (RuntimeException e) { + if (!isAuthExpired(e) || !usesCatalogIdentity(context)) { + throw e; + } + reauthenticate(attemptedOn, e); + return op.get(); + } + } + + private void runWithAuthRecovery(SessionContext context, Runnable op) { + withAuthRecovery(context, () -> { + op.run(); + return null; + }); + } + + /** + * Rebuilds the delegate (fresh client, fresh token) and closes the wedged one. Synchronized so that + * concurrent 401s coalesce into a single rebuild: a thread whose failed attempt ran against an + * already-replaced delegate skips the rebuild and just retries on the fresh one. + */ + private synchronized void reauthenticate(RESTSessionCatalog attemptedOn, RuntimeException cause) { + if (delegate != attemptedOn) { + return; + } + LOG.warn("Iceberg REST catalog {} rejected its cached credential (401 Not Authorized) and the client " + + "cannot recover it internally. Rebuilding the REST client to force re-authentication, " + + "then retrying the request once.", name(), cause); + RESTSessionCatalog replacement = delegateBuilder.get(); + RESTSessionCatalog wedged = delegate; + delegate = replacement; + try { + wedged.close(); + } catch (IOException | RuntimeException e) { + LOG.warn("Failed to close the replaced Iceberg REST client of catalog {}", name(), e); + } + } + + /** + * A request made with a per-user delegated credential authenticates as that user, not as the catalog; + * a 401 there is that token's problem and must not rebuild (or leak into) the shared client. + */ + private static boolean usesCatalogIdentity(SessionContext context) { + return context == null || context.credentials() == null || context.credentials().isEmpty(); + } + + private static boolean isAuthExpired(Throwable t) { + return ExceptionUtils.getThrowableList(t).stream().anyMatch(c -> c instanceof NotAuthorizedException); + } + + // ---- SessionCatalog ---- + + @Override + public void initialize(String name, Map properties) { + super.initialize(name, properties); + delegate.initialize(name, properties); + } + + @Override + public String name() { + return delegate.name(); + } + + @Override + public Map properties() { + return delegate.properties(); + } + + @Override + public List listTables(SessionContext context, Namespace ns) { + return withAuthRecovery(context, () -> delegate.listTables(context, ns)); + } + + @Override + public Catalog.TableBuilder buildTable(SessionContext context, TableIdentifier ident, Schema schema) { + return withAuthRecovery(context, () -> delegate.buildTable(context, ident, schema)); + } + + @Override + public Table registerTable(SessionContext context, TableIdentifier ident, String metadataFileLocation) { + return withAuthRecovery(context, () -> delegate.registerTable(context, ident, metadataFileLocation)); + } + + @Override + public boolean tableExists(SessionContext context, TableIdentifier ident) { + return withAuthRecovery(context, () -> delegate.tableExists(context, ident)); + } + + @Override + public Table loadTable(SessionContext context, TableIdentifier ident) { + return withAuthRecovery(context, () -> delegate.loadTable(context, ident)); + } + + @Override + public boolean dropTable(SessionContext context, TableIdentifier ident) { + return withAuthRecovery(context, () -> delegate.dropTable(context, ident)); + } + + @Override + public boolean purgeTable(SessionContext context, TableIdentifier ident) { + return withAuthRecovery(context, () -> delegate.purgeTable(context, ident)); + } + + @Override + public void renameTable(SessionContext context, TableIdentifier from, TableIdentifier to) { + runWithAuthRecovery(context, () -> delegate.renameTable(context, from, to)); + } + + @Override + public void invalidateTable(SessionContext context, TableIdentifier ident) { + runWithAuthRecovery(context, () -> delegate.invalidateTable(context, ident)); + } + + @Override + public void createNamespace(SessionContext context, Namespace namespace, Map metadata) { + runWithAuthRecovery(context, () -> delegate.createNamespace(context, namespace, metadata)); + } + + @Override + public List listNamespaces(SessionContext context, Namespace namespace) { + return withAuthRecovery(context, () -> delegate.listNamespaces(context, namespace)); + } + + @Override + public Map loadNamespaceMetadata(SessionContext context, Namespace namespace) { + return withAuthRecovery(context, () -> delegate.loadNamespaceMetadata(context, namespace)); + } + + @Override + public boolean dropNamespace(SessionContext context, Namespace namespace) { + return withAuthRecovery(context, () -> delegate.dropNamespace(context, namespace)); + } + + @Override + public boolean updateNamespaceMetadata(SessionContext context, Namespace namespace, + Map updates, Set removals) { + return withAuthRecovery(context, () -> delegate.updateNamespaceMetadata(context, namespace, updates, + removals)); + } + + @Override + public boolean namespaceExists(SessionContext context, Namespace namespace) { + return withAuthRecovery(context, () -> delegate.namespaceExists(context, namespace)); + } + + // ---- ViewSessionCatalog ---- + + @Override + public List listViews(SessionContext context, Namespace namespace) { + return withAuthRecovery(context, () -> delegate.listViews(context, namespace)); + } + + @Override + public View loadView(SessionContext context, TableIdentifier identifier) { + return withAuthRecovery(context, () -> delegate.loadView(context, identifier)); + } + + @Override + public boolean viewExists(SessionContext context, TableIdentifier identifier) { + return withAuthRecovery(context, () -> delegate.viewExists(context, identifier)); + } + + @Override + public ViewBuilder buildView(SessionContext context, TableIdentifier identifier) { + return withAuthRecovery(context, () -> delegate.buildView(context, identifier)); + } + + @Override + public boolean dropView(SessionContext context, TableIdentifier identifier) { + return withAuthRecovery(context, () -> delegate.dropView(context, identifier)); + } + + @Override + public void renameView(SessionContext context, TableIdentifier from, TableIdentifier to) { + runWithAuthRecovery(context, () -> delegate.renameView(context, from, to)); + } + + @Override + public void invalidateView(SessionContext context, TableIdentifier identifier) { + runWithAuthRecovery(context, () -> delegate.invalidateView(context, identifier)); + } + + @Override + public void close() throws IOException { + delegate.close(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java index b67a5911b64384..54a791e7e18133 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java @@ -30,12 +30,21 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.MetricsModes; +import org.apache.iceberg.MetricsUtil; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.BinaryUtil; +import org.apache.iceberg.util.UnicodeUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -61,6 +70,12 @@ public static WriteResult convertToWriterResult( // Get table specification information PartitionSpec spec = table.spec(); FileFormat fileFormat = IcebergUtils.getFileFormat(table); + MetricsConfig metricsConfig = MetricsConfig.forTable(table); + Schema schema = table.schema(); + if (IcebergUtils.getFormatVersion(table) >= IcebergUtils.ICEBERG_ROW_LINEAGE_MIN_VERSION) { + // Rewrite and merge writers emit v3 lineage columns that are absent from the table schema. + schema = IcebergUtils.appendRowLineageFieldsForV3(schema); + } for (TIcebergCommitData commitData : commitDataList) { //get the files path @@ -70,7 +85,7 @@ public static WriteResult convertToWriterResult( long fileSize = commitData.getFileSize(); long recordCount = commitData.getRowCount(); CommonStatistics stat = new CommonStatistics(recordCount, DEFAULT_FILE_COUNT, fileSize); - Metrics metrics = buildDataFileMetrics(table, fileFormat, commitData); + Metrics metrics = buildDataFileMetrics(commitData, schema, metricsConfig, fileFormat); Optional partitionData = Optional.empty(); //get and check partitionValues when table is partitionedTable if (spec.isPartitioned()) { @@ -153,7 +168,9 @@ private static PartitionData convertToPartitionData( return partitionData; } - private static Metrics buildDataFileMetrics(Table table, FileFormat fileFormat, TIcebergCommitData commitData) { + private static Metrics buildDataFileMetrics( + TIcebergCommitData commitData, Schema schema, MetricsConfig metricsConfig, FileFormat fileFormat) { + Map fieldParents = TypeUtil.indexParents(schema.asStruct()); Map columnSizes = new HashMap<>(); Map valueCounts = new HashMap<>(); Map nullValueCounts = new HashMap<>(); @@ -178,8 +195,100 @@ private static Metrics buildDataFileMetrics(Table table, FileFormat fileFormat, } } - return new Metrics(commitData.getRowCount(), columnSizes, valueCounts, - nullValueCounts, null, lowerBounds, upperBounds); + // Physical file stats may contain every column, but manifest metrics must honor the table's metadata policy. + return new Metrics(commitData.getRowCount(), + filterDisabledMetrics(columnSizes, schema, metricsConfig), + filterLogicalMetrics(valueCounts, schema, metricsConfig, fieldParents), + filterLogicalMetrics(nullValueCounts, schema, metricsConfig, fieldParents), + null, + filterBounds(lowerBounds, schema, metricsConfig, fieldParents, fileFormat, true), + filterBounds(upperBounds, schema, metricsConfig, fieldParents, fileFormat, false)); + } + + private static Map filterDisabledMetrics( + Map metrics, Schema schema, MetricsConfig metricsConfig) { + Map filteredMetrics = new HashMap<>(); + metrics.forEach((fieldId, value) -> { + if (MetricsUtil.metricsMode(schema, metricsConfig, fieldId) != MetricsModes.None.get()) { + filteredMetrics.put(fieldId, value); + } + }); + return filteredMetrics; + } + + private static Map filterLogicalMetrics( + Map metrics, Schema schema, MetricsConfig metricsConfig, + Map fieldParents) { + Map filteredMetrics = new HashMap<>(); + metrics.forEach((fieldId, value) -> { + // Definition-level values below list/map do not represent logical element counts. + if (!isInRepeatedField(fieldId, schema, fieldParents) + && MetricsUtil.metricsMode(schema, metricsConfig, fieldId) != MetricsModes.None.get()) { + filteredMetrics.put(fieldId, value); + } + }); + return filteredMetrics; + } + + private static Map filterBounds( + Map bounds, Schema schema, MetricsConfig metricsConfig, + Map fieldParents, FileFormat fileFormat, boolean lowerBound) { + Map filteredBounds = new HashMap<>(); + bounds.forEach((fieldId, value) -> { + if (isInRepeatedField(fieldId, schema, fieldParents)) { + return; + } + MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema, metricsConfig, fieldId); + if (mode == MetricsModes.None.get() || mode == MetricsModes.Counts.get()) { + return; + } + + ByteBuffer filteredValue = value; + if (mode instanceof MetricsModes.Truncate) { + Type type = schema.findType(fieldId); + int length = ((MetricsModes.Truncate) mode).length(); + // Truncated upper bounds must round up so file pruning cannot exclude matching values. + filteredValue = truncateBound(type, value, length, fileFormat, lowerBound); + } + if (filteredValue != null) { + filteredBounds.put(fieldId, filteredValue); + } + }); + return filteredBounds; + } + + private static boolean isInRepeatedField( + int fieldId, Schema schema, Map fieldParents) { + Integer parentId = fieldId; + while ((parentId = fieldParents.get(parentId)) != null) { + Types.NestedField parent = schema.findField(parentId); + if (parent != null && !parent.type().isStructType()) { + return true; + } + } + return false; + } + + private static ByteBuffer truncateBound( + Type type, ByteBuffer value, int length, FileFormat fileFormat, boolean lowerBound) { + switch (type.typeId()) { + case STRING: + String stringValue = Conversions.fromByteBuffer(type, value).toString(); + String truncatedString = lowerBound + ? UnicodeUtil.truncateStringMin(stringValue, length) + : UnicodeUtil.truncateStringMax(stringValue, length); + // ORC keeps the full maximum when no safe truncated successor exists. + if (!lowerBound && truncatedString == null && fileFormat == FileFormat.ORC) { + return value; + } + return truncatedString == null ? null : Conversions.toByteBuffer(type, truncatedString); + case BINARY: + return lowerBound + ? BinaryUtil.truncateBinaryMin(value, length) + : BinaryUtil.truncateBinaryMax(value, length); + default: + return value; + } } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java index 32de4ebfdd9c0c..083409adda136c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilter.java @@ -54,16 +54,46 @@ public static PositionDelete createPositionDelete(DeleteFile deleteFile) { String deleteFilePath = deleteFile.path().toString(); if (deleteFile.format() == FileFormat.PUFFIN) { + long fileSize = deleteFile.fileSizeInBytes(); + Long contentOffset = deleteFile.contentOffset(); + Long contentLength = deleteFile.contentSizeInBytes(); + validateDeletionVectorMetadata(deleteFilePath, fileSize, contentOffset, contentLength); // The content_offset and content_size_in_bytes fields are used to reference // a specific blob for direct access to a deletion vector. return new DeletionVector(deleteFilePath, positionLowerBound.orElse(-1L), positionUpperBound.orElse(-1L), - deleteFile.fileSizeInBytes(), deleteFile.contentOffset(), deleteFile.contentSizeInBytes()); + fileSize, contentOffset, contentLength); } else { return new PositionDelete(deleteFilePath, positionLowerBound.orElse(-1L), positionUpperBound.orElse(-1L), deleteFile.fileSizeInBytes(), deleteFile.format()); } } + static void validateDeletionVectorMetadata( + String deleteFilePath, long fileSize, Long contentOffset, Long contentLength) { + if (contentOffset == null || contentLength == null) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata misses content offset or length: %s", deleteFilePath)); + } + if (fileSize < 0 || contentOffset < 0 || contentLength < 0) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata must be non-negative, file: %s, file size: %d, " + + "content offset: %d, content length: %d", + deleteFilePath, fileSize, contentOffset, contentLength)); + } + if (contentOffset > Long.MAX_VALUE - contentLength) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata range overflows, file: %s, content offset: %d, " + + "content length: %d", + deleteFilePath, contentOffset, contentLength)); + } + if (contentOffset + contentLength > fileSize) { + throw new IllegalArgumentException(String.format( + "Iceberg deletion vector metadata range exceeds file size, file: %s, file size: %d, " + + "content offset: %d, content length: %d", + deleteFilePath, fileSize, contentOffset, contentLength)); + } + } + public static EqualityDelete createEqualityDelete(String deleteFilePath, List fieldIds, long fileSize, FileFormat fileformat) { // todo: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 70ac1d26b108fe..34efa48364f821 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -25,7 +25,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; @@ -41,12 +40,15 @@ import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.iceberg.cache.IcebergManifestCacheLoader; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.iceberg.profile.IcebergMetricsReporter; import org.apache.doris.datasource.iceberg.source.IcebergDeleteFileFilter.EqualityDelete; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.persist.gson.GsonUtils; @@ -63,7 +65,6 @@ import org.apache.doris.thrift.TIcebergDeleteFileDesc; import org.apache.doris.thrift.TIcebergFileDesc; import org.apache.doris.thrift.TPlanNode; -import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.annotations.VisibleForTesting; @@ -94,8 +95,8 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.SplittableScanTask; import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; +import org.apache.iceberg.expressions.Binder; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.InclusiveMetricsEvaluator; @@ -103,11 +104,8 @@ import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; -import org.apache.iceberg.mapping.MappedField; -import org.apache.iceberg.mapping.MappedFields; -import org.apache.iceberg.mapping.NameMapping; -import org.apache.iceberg.mapping.NameMappingParser; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.util.ScanTaskUtil; import org.apache.iceberg.util.SerializationUtil; @@ -121,17 +119,20 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; public class IcebergScanNode extends FileQueryScanNode { public static final int MIN_DELETE_FILE_SUPPORT_VERSION = 2; + static final int ICEBERG_SCAN_SEMANTICS_VERSION = 1; private static final Logger LOG = LogManager.getLogger(IcebergScanNode.class); private IcebergSource source; @@ -265,38 +266,14 @@ protected void doInitialize() throws UserException { super.doInitialize(); } - /** - * Extract name mapping from Iceberg table properties. - * Returns a map from field ID to list of mapped names. - */ - private Map> extractNameMapping() { - Map> result = new HashMap<>(); - try { - String nameMappingJson = icebergTable.properties().get(TableProperties.DEFAULT_NAME_MAPPING); - if (nameMappingJson != null && !nameMappingJson.isEmpty()) { - NameMapping mapping = NameMappingParser.fromJson(nameMappingJson); - if (mapping != null) { - // Extract mappings from NameMapping - // NameMapping contains field mappings, we need to convert them to our format - extractMappingsFromNameMapping(mapping.asMappedFields(), result); - } - } - } catch (Exception e) { - // If name mapping parsing fails, continue without it - LOG.warn("Failed to parse name mapping from Iceberg table properties", e); - } - return result; - } - - private void extractMappingsFromNameMapping(MappedFields mappingFields, Map> result) { - if (mappingFields == null) { - return; - } - for (MappedField mappedField : mappingFields.fields()) { - result.put(mappedField.id(), new ArrayList<>(mappedField.names())); - extractMappingsFromNameMapping(mappedField.nestedMapping(), result); + private Optional>> extractNameMapping() { + Optional snapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + if (snapshot.isPresent() && snapshot.get() instanceof IcebergMvccSnapshot) { + // The mapping must come from the same metadata generation as the pinned schema; a + // property-only refresh can otherwise change alias semantics within one statement. + return ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue().getNameMapping(); } - + return IcebergUtils.getNameMapping(icebergTable); } @Override @@ -325,6 +302,8 @@ private void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSpli rangeDesc.unsetColumnsFromPathIsNull(); return; } + // update for every split file format + rangeDesc.setFormatType(toTFileFormatType(icebergSplit.getSplitFileFormat())); if (tableLevelPushDownCount) { tableFormatFileDesc.setTableLevelRowCount(icebergSplit.getTableLevelRowCount()); } else { @@ -494,6 +473,15 @@ private void setDeleteFileFormat(TIcebergDeleteFileDesc deleteFileDesc, FileForm } } + private TFileFormatType toTFileFormatType(FileFormat fileFormat) { + if (fileFormat == FileFormat.PARQUET) { + return TFileFormatType.FORMAT_PARQUET; + } else if (fileFormat == FileFormat.ORC) { + return TFileFormatType.FORMAT_ORC; + } + throw new UnsupportedOperationException("Unsupported Iceberg data file format: " + fileFormat); + } + private String getDeleteFileContentType(int content) { // Iceberg file type: 0: data, 1: position delete, 2: equality delete, 3: deletion vector switch (content) { @@ -517,14 +505,23 @@ private List getOrderedPathPartitionKeys() { public void createScanRangeLocations() throws UserException { super.createScanRangeLocations(); + enableCurrentIcebergScanSemantics(); // Extract name mapping from Iceberg table properties - Map> nameMapping = extractNameMapping(); + Optional>> nameMapping = extractNameMapping(); // Equality-delete keys are hidden scan dependencies and need not appear in the query // projection. Both scanners need the complete current schema to resolve field ids, // historical names, types, and initial defaults when an old data file lacks such a key. ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), - nameMapping, getBase64EncodedInitialDefaultsForScan()); + nameMapping.orElse(Collections.emptyMap()), nameMapping.isPresent(), + getBase64EncodedInitialDefaultsForScan()); + } + + @VisibleForTesting + void enableCurrentIcebergScanSemantics() { + // This explicit capability is the rollout boundary: old FE plans must keep legacy values + // when fragments run on a mixture of old and new BEs. + params.setIcebergScanSemanticsVersion(ICEBERG_SCAN_SEMANTICS_VERSION); } @VisibleForTesting @@ -535,17 +532,22 @@ Map getBase64EncodedInitialDefaultsForScan() throws UserExcepti // schema that produced source.getTargetTable().getColumns() to keep defaults aligned. return IcebergUtils.getBase64EncodedInitialDefaults(icebergTable.schema()); } - TableScan tableScan = createTableScan(); - Snapshot snapshot = tableScan.snapshot(); - // TableScan.schema() starts from the table's current schema even for useSnapshot/useRef. - // Resolve the selected snapshot's schema id explicitly so this metadata describes the same - // snapshot as source.getTargetTable().getColumns(). Otherwise a later type change can make - // BE decode a historical non-binary default as Base64, or fail to decode a binary default. - Schema scanSchema = snapshot == null - ? tableScan.schema() - : tableScan.table().schemas().get(snapshot.schemaId()); + IcebergTableQueryInfo selectedSnapshot = getSpecifiedSnapshot(); + Optional mvccSnapshot = MvccUtil.getSnapshotFromContext(source.getTargetTable()); + Schema scanSchema = null; + if (mvccSnapshot.isPresent() && mvccSnapshot.get() instanceof IcebergMvccSnapshot) { + long schemaId = ((IcebergMvccSnapshot) mvccSnapshot.get()) + .getSnapshotCacheValue().getSnapshot().getSchemaId(); + scanSchema = icebergTable.schemas().get(Math.toIntExact(schemaId)); + } else { + scanSchema = selectedSnapshot == null + ? icebergTable.schema() + : icebergTable.schemas().get(selectedSnapshot.getSchemaId()); + } + // A branch can expose a schema newer than its data snapshot. The statement-pinned schema + // produced the target columns, so default markers must not be recomputed from that snapshot. return IcebergUtils.getBase64EncodedInitialDefaults( - Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan snapshot is null")); + Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan is null")); } @Override @@ -674,11 +676,70 @@ public TableScan createTableScan() throws UserException { this.pushdownIcebergPredicates.add(predicate.toString()); } + // Doris reads normal Iceberg table files in BE and applies column pruning through scan range params. + // System tables are different: Iceberg SDK DataTask materializes rows using the projected scan + // schema. Keep Doris file slots in the same order as the JNI reader's required fields. + if (isSystemTable) { + Schema projectedSchema = getSystemTableProjectedSchema(expressions, scan.isCaseSensitive()); + Preconditions.checkState(!projectedSchema.columns().isEmpty(), + "Iceberg system table scan must materialize at least one file slot"); + scan = scan.project(projectedSchema); + } + icebergTableScan = scan.planWith(source.getCatalog().getThreadPoolWithPreAuth()); return icebergTableScan; } + @VisibleForTesting + Schema getSystemTableProjectedSchema(List expressions, boolean caseSensitive) + throws UserException { + List projectedFields = new ArrayList<>(); + Set projectedFieldIds = new HashSet<>(); + List partitionKeys = getPathPartitionKeys(); + for (SlotDescriptor slot : desc.getSlots()) { + Column column = slot.getColumn(); + String columnName = column.getName(); + if (!isFileSlot(classifyColumn(slot, partitionKeys))) { + continue; + } + + NestedField field = caseSensitive + ? icebergTable.schema().findField(columnName) + : icebergTable.schema().caseInsensitiveFindField(columnName); + if (field == null) { + throw new UserException("Column " + columnName + " not found in Iceberg system table schema"); + } + if (projectedFieldIds.add(field.fieldId())) { + projectedFields.add(field); + } + } + + Set filterFieldIds = Binder.boundReferences( + icebergTable.schema().asStruct(), expressions, caseSensitive); + for (Integer fieldId : filterFieldIds) { + NestedField field = getTopLevelSystemTableField(fieldId); + if (field == null) { + throw new UserException( + "Column with field id " + fieldId + " not found in Iceberg system table schema"); + } + if (!projectedFieldIds.contains(field.fieldId())) { + throw new UserException("Iceberg system table filter column " + field.name() + + " is not materialized by the planner"); + } + } + return new Schema(projectedFields); + } + + private NestedField getTopLevelSystemTableField(int fieldId) { + for (NestedField field : icebergTable.schema().columns()) { + if (field.fieldId() == fieldId || TypeUtil.getProjectedIds(field.type()).contains(fieldId)) { + return field; + } + } + return null; + } + private CloseableIterable planFileScanTask(TableScan scan) { if (!IcebergUtils.isManifestCacheEnabled(source.getCatalog())) { return splitFiles(scan); @@ -945,6 +1006,7 @@ private Split createIcebergSplit(FileScanTask fileScanTask) { storagePropertiesMap, new ArrayList<>(), originalPath); + split.setSplitFileFormat(dataFile.format()); if (formatVersion >= 3) { // -1 means that this table was just upgraded from v2 to v3. // _row_id and _last_updated_sequence_number column is NULL. @@ -1003,10 +1065,14 @@ private Split createIcebergPositionDeleteSysSplit(PositionDeletesScanTask task) split.setPositionDeleteFileFormat(getNativePositionDeleteFileFormat(deleteFile.format())); split.setPositionDeleteOriginalPath(originalPath); if (deleteFile.format() == FileFormat.PUFFIN) { + Long contentOffset = deleteFile.contentOffset(); + Long contentLength = deleteFile.contentSizeInBytes(); + IcebergDeleteFileFilter.validateDeletionVectorMetadata( + originalPath, deleteFile.fileSizeInBytes(), contentOffset, contentLength); split.setPositionDeleteContent(IcebergDeleteFileFilter.DeletionVector.type()); split.setPositionDeleteReferencedDataFilePath(deleteFile.referencedDataFile()); - split.setPositionDeleteContentOffset(deleteFile.contentOffset()); - split.setPositionDeleteContentSizeInBytes(deleteFile.contentSizeInBytes()); + split.setPositionDeleteContentOffset(contentOffset); + split.setPositionDeleteContentSizeInBytes(contentLength); } else { split.setPositionDeleteContent(IcebergDeleteFileFilter.PositionDelete.type()); } @@ -1199,7 +1265,7 @@ private List doGetSystemTableSplits() throws UserException { private boolean isPositionDeletesSystemTable() { TableIf targetTable = source.getTargetTable(); return targetTable instanceof IcebergSysExternalTable - && "position_deletes".equalsIgnoreCase(((IcebergSysExternalTable) targetTable).getSysTableType()); + && ((IcebergSysExternalTable) targetTable).isPositionDeletesTable(); } private List doGetPositionDeletesSystemTableSplits() throws UserException { @@ -1275,8 +1341,7 @@ public boolean isBatchMode() { if (cached != null) { return cached; } - TPushAggOp aggOp = getPushDownAggNoGroupingOp(); - if (aggOp.equals(TPushAggOp.COUNT)) { + if (isTableLevelCountStarPushdown()) { try { countFromSnapshot = getCountFromSnapshot(); } catch (UserException e) { @@ -1367,16 +1432,8 @@ public TFileFormatType getFileFormatType() throws UserException { if (isSystemTable) { return TFileFormatType.FORMAT_JNI; } - TFileFormatType type; - String icebergFormat = source.getFileFormat(); - if (icebergFormat.equalsIgnoreCase("parquet")) { - type = TFileFormatType.FORMAT_PARQUET; - } else if (icebergFormat.equalsIgnoreCase("orc")) { - type = TFileFormatType.FORMAT_ORC; - } else { - throw new DdlException(String.format("Unsupported format name: %s for iceberg table.", icebergFormat)); - } - return type; + // for table level file format + return toTFileFormatType(IcebergUtils.getFileFormat(icebergTable)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java index 59b4f483f019e3..eeeff694b8ebce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSplit.java @@ -24,6 +24,7 @@ import lombok.Data; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; import java.util.ArrayList; import java.util.Collections; @@ -53,6 +54,8 @@ public class IcebergSplit extends FileSplit { private Long firstRowId = null; private Long lastUpdatedSequenceNumber = null; private String serializedSplit; + // maybe mixed file format type in one table. so need record it for every split + private FileFormat splitFileFormat; private boolean positionDeleteSystemTableSplit = false; private TFileFormatType positionDeleteFileFormat; private int positionDeleteContent; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java index 4f340bebed4732..fec623e88f2b91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java @@ -39,12 +39,9 @@ public class JdbcClickHouseClient extends JdbcClient { protected JdbcClickHouseClient(JdbcClientConfig jdbcClientConfig) { super(jdbcClientConfig); try (Connection conn = getConnection()) { - String jdbcUrl = conn.getMetaData().getURL(); - if (!isNewClickHouseDriver(getJdbcDriverVersion())) { - this.databaseTermIsCatalog = false; - } else { - this.databaseTermIsCatalog = "catalog".equalsIgnoreCase(getDatabaseTermFromUrl(jdbcUrl)); - } + DatabaseMetaData databaseMetaData = conn.getMetaData(); + this.databaseTermIsCatalog = isDatabaseTermCatalog( + databaseMetaData, databaseMetaData.getDriverVersion()); } catch (SQLException e) { throw new JdbcClientException("Failed to initialize JdbcClickHouseClient: %s", e.getMessage()); } @@ -123,7 +120,8 @@ protected String getCatalogName(Connection conn) throws SQLException { @Override protected String[] getTableTypes() { - return new String[] {"TABLE", "VIEW", "SYSTEM TABLE"}; + // ClickHouse JDBC V2 filters engines by these vendor-specific table type names. + return new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE", "MATERIALIZED VIEW"}; } @Override @@ -231,14 +229,9 @@ private static boolean isNewClickHouseDriver(String driverVersion) { } } - /** - * Extract databaseterm parameters from the jdbc url. - */ - private String getDatabaseTermFromUrl(String jdbcUrl) { - if (jdbcUrl != null && jdbcUrl.toLowerCase().contains("databaseterm=schema")) { - return "schema"; - } - return "catalog"; + static boolean isDatabaseTermCatalog(DatabaseMetaData databaseMetaData, String driverVersion) + throws SQLException { + return isNewClickHouseDriver(driverVersion) && databaseMetaData.supportsCatalogsInDataManipulation(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java index b7f6331306861c..f513eb10fca7a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.operations; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.info.ColumnPosition; import org.apache.doris.catalog.info.CreateOrReplaceBranchInfo; @@ -226,6 +227,15 @@ default void addColumn(ExternalTable dorisTable, Column column, ColumnPosition p throw new UnsupportedOperationException("Add column operation is not supported for this table type."); } + default void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, + long updateTime) throws UserException { + if (!columnPath.isNested()) { + addColumn(dorisTable, column, position, updateTime); + return; + } + throw new UnsupportedOperationException("Nested add column operation is not supported for this table type."); + } + /** * add columns for external table * @@ -250,6 +260,15 @@ default void dropColumn(ExternalTable dorisTable, String columnName, long update throw new UnsupportedOperationException("Drop column operation is not supported for this table type."); } + default void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long updateTime) + throws UserException { + if (!columnPath.isNested()) { + dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); + return; + } + throw new UnsupportedOperationException("Nested drop column operation is not supported for this table type."); + } + /** * rename column for external table * @@ -263,6 +282,15 @@ default void renameColumn(ExternalTable dorisTable, String oldName, String newNa throw new UnsupportedOperationException("Rename column operation is not supported for this table type."); } + default void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String newName, long updateTime) + throws UserException { + if (!columnPath.isNested()) { + renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); + return; + } + throw new UnsupportedOperationException("Nested rename column operation is not supported for this table type."); + } + /** * update column for external table * @@ -276,6 +304,30 @@ default void modifyColumn(ExternalTable dorisTable, Column column, ColumnPositio throw new UnsupportedOperationException("Modify column operation is not supported for this table type."); } + default void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, + long updateTime) throws UserException { + if (!columnPath.isNested()) { + modifyColumn(dorisTable, column, position, updateTime); + return; + } + throw new UnsupportedOperationException("Nested modify column operation is not supported for this table type."); + } + + /** + * modify column comment for external table + * + * @param dorisTable + * @param columnPath + * @param comment + * @param updateTime + * @throws UserException + */ + default void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) + throws UserException { + throw new UnsupportedOperationException( + "Modify column comment operation is not supported for this table type."); + } + /** * reorder columns for external table * diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index c96ffa6146cf2c..2787ba12d3802f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -29,7 +29,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.hive.HiveUtil; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TArrayField; @@ -77,6 +76,7 @@ import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.PartitionPathUtils; import org.apache.paimon.utils.Projection; import org.apache.paimon.utils.RowDataToObjectArrayConverter; @@ -91,6 +91,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -162,47 +163,50 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo List types = partitionColumns.stream() .map(Column::getType) .collect(Collectors.toList()); - Map columnNameToType = partitionColumns.stream() - .collect(Collectors.toMap(Column::getName, Column::getType)); for (Partition partition : paimonPartitions) { Map spec = partition.spec(); - StringBuilder sb = new StringBuilder(); - for (Map.Entry entry : spec.entrySet()) { - sb.append(entry.getKey()).append("="); + // Paimon partition specs contain logical values, which may include path separators. + // Build partition values directly instead of parsing them as a Hive partition path. + List partitionValues = Lists.newArrayListWithExpectedSize(partitionColumns.size()); + LinkedHashMap orderedPartitionSpec = new LinkedHashMap<>(); + for (Column partitionColumn : partitionColumns) { + String partitionColumnName = partitionColumn.getName(); + String partitionValue = spec.get(partitionColumnName); // When partition.legacy-name = true (default), Paimon stores DATE type as days since // 1970-01-01 (epoch integer), so we need to convert the integer to a date string. // When partition.legacy-name = false, the value is already a human read date string. - if (legacyPartitionName - && columnNameToType.getOrDefault(entry.getKey(), Type.NULL).isDateV2()) { - sb.append(DateTimeUtils.formatDate(Integer.parseInt(entry.getValue()))).append("/"); - } else { - sb.append(entry.getValue()).append("/"); + if (legacyPartitionName && partitionColumn.getType().isDateV2()) { + partitionValue = DateTimeUtils.formatDate(Integer.parseInt(partitionValue)); } + partitionValues.add(partitionValue); + orderedPartitionSpec.put(partitionColumnName, partitionValue); } - if (sb.length() > 0) { - sb.deleteCharAt(sb.length() - 1); - } - String partitionName = sb.toString(); - nameToPartition.put(partitionName, partition); + String partitionPath = PartitionPathUtils.generatePartitionPath(orderedPartitionSpec); + String partitionName = partitionPath.substring(0, partitionPath.length() - 1); + Partition previousPartition = nameToPartition.putIfAbsent(partitionName, partition); + Preconditions.checkState(previousPartition == null, + "Duplicate Paimon partition name: " + partitionName); + PartitionItem partitionItem; try { // partition values return by paimon api, may have problem, // to avoid affecting the query, we catch exceptions here - nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, types)); + partitionItem = toListPartitionItem(partitionValues, types); } catch (Exception e) { LOG.warn("toListPartitionItem failed, partitionColumns: {}, partitionValues: {}", partitionColumns, partition.spec(), e); + continue; } + PartitionItem previousPartitionItem = nameToPartitionItem.putIfAbsent(partitionName, partitionItem); + Preconditions.checkState(previousPartitionItem == null, + "Duplicate Paimon partition item name: " + partitionName); } return partitionInfo; } - public static ListPartitionItem toListPartitionItem(String partitionName, List types) + public static ListPartitionItem toListPartitionItem(List partitionValues, List types) throws AnalysisException { - // Partition name will be in format: nation=cn/city=beijing - // parse it to get values "cn" and "beijing" - List partitionValues = HiveUtil.toPartitionValues(partitionName); - Preconditions.checkState(partitionValues.size() == types.size(), partitionName + " vs. " + types); + Preconditions.checkState(partitionValues.size() == types.size(), partitionValues + " vs. " + types); List values = Lists.newArrayListWithExpectedSize(types.size()); for (String partitionValue : partitionValues) { // null will in partition 'null' diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index da3e50a6be46e4..990e79923f0a93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -48,7 +48,6 @@ import org.apache.doris.thrift.TPaimonDeletionFileDesc; import org.apache.doris.thrift.TPaimonFileDesc; import org.apache.doris.thrift.TPaimonReaderType; -import org.apache.doris.thrift.TPushAggOp; import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.annotations.VisibleForTesting; @@ -405,7 +404,9 @@ public List getSplits(int numBackends) throws UserException { ++paimonSplitNum; } - boolean applyCountPushdown = getPushDownAggNoGroupingOp() == TPushAggOp.COUNT; + // Merged row counts contain only COUNT(*) semantics. COUNT(col) must keep every DataSplit + // because BE will read the argument column to account for NULL and schema-mapping rules. + boolean applyCountPushdown = isTableLevelCountStarPushdown(); // Used to avoid repeatedly calculating partition info map for the same // partition data. // And for counting the number of selected partitions for this paimon table. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java index d420e40724fed3..d276b6c317d749 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java @@ -21,6 +21,7 @@ import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.iceberg.IcebergDelegatedCredentialUtils; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; +import org.apache.doris.datasource.iceberg.ReauthenticatingRestSessionCatalog; import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import org.apache.doris.datasource.property.common.IcebergAwsClientCredentialsProperties; import org.apache.doris.datasource.property.storage.S3Properties; @@ -32,6 +33,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.BaseViewSessionCatalog; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SessionCatalog; import org.apache.iceberg.rest.RESTSessionCatalog; @@ -41,6 +43,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.util.Strings; +import java.io.Closeable; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -66,7 +69,10 @@ public class IcebergRestProperties extends AbstractIcebergProperties { // asCatalog(SessionContext) / asViewCatalog(SessionContext), without reflecting RESTCatalog's private // sessionCatalog field. This is the single underlying catalog shared by the default and user-session // paths; IcebergMetadataOps reads it via getRestSessionCatalog() and owns no other REST catalog. - private RESTSessionCatalog restSessionCatalog; + // When the catalog authenticates with its own identity this is a ReauthenticatingRestSessionCatalog + // wrapping the RESTSessionCatalog, so an expired/rejected token is recovered by rebuilding the client + // instead of failing every request until the FE restarts. + private BaseViewSessionCatalog restSessionCatalog; @Getter @ConnectorProperty(names = {"iceberg.rest.uri", "uri"}, @@ -241,7 +247,22 @@ protected Catalog initCatalog(String catalogName, Map catalogPro // it ourselves keeps that capability without reflection. The "type" key is dropped because the // Iceberg SDK rejects "type" together with a concrete catalog impl. catalogProps.remove(CatalogUtil.ICEBERG_CATALOG_TYPE); - this.restSessionCatalog = buildRestSessionCatalog(catalogName, catalogProps, configuration); + RESTSessionCatalog rawSessionCatalog = buildRestSessionCatalog(catalogName, catalogProps, configuration); + if (sessionContext == null || !sessionContext.hasDelegatedCredential()) { + // The catalog authenticates with its own identity (e.g. an oauth2 client credential). If that + // credential's token expires and the client cannot refresh it (auth server briefly unreachable at + // refresh time), the RESTSessionCatalog is left rejecting every request with a 401 until the FE + // restarts. Wrap it so a 401 rebuilds the client (fresh token) and retries once. The rebuild + // supplier re-resolves from the same catalog-identity properties, so it can never capture a + // per-user delegated credential. + Map frozenProps = Collections.unmodifiableMap(new HashMap<>(catalogProps)); + this.restSessionCatalog = new ReauthenticatingRestSessionCatalog(rawSessionCatalog, + () -> buildRestSessionCatalog(catalogName, frozenProps, configuration)); + } else { + // Catalog initialization under a per-user delegated credential: recovery must not re-mint the + // client with that user's token, so no wrapper — behavior is unchanged from before. + this.restSessionCatalog = rawSessionCatalog; + } // The default (non-delegated) Catalog is asCatalog(empty), identical to what RESTCatalog exposes. return restSessionCatalog.asCatalog(SessionCatalog.SessionContext.createEmpty()); } @@ -263,7 +284,7 @@ protected RESTSessionCatalog buildRestSessionCatalog(String catalogName, Map getSplits(int numBackends) throws UserException { List fileStatuses = tableValuedFunction.getFileStatuses(); - // Push down count optimization. + // Avoid splitting only for table-level COUNT(*). COUNT(column) still reads column data. boolean needSplit = true; - if (getPushDownAggNoGroupingOp() == TPushAggOp.COUNT) { + if (isTableLevelCountStarPushdown()) { int parallelNum = sessionVariable.getParallelExecInstanceNum(scanContext.getClusterName()); int totalFileNum = fileStatuses.size(); needSplit = FileSplitter.needSplitForCountPushdown(parallelNum, numBackends, totalFileNum); diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/SpiSwitchingFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/SpiSwitchingFileSystem.java index 9257352238caae..dac4c9071786e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/SpiSwitchingFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/SpiSwitchingFileSystem.java @@ -20,6 +20,7 @@ import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.filesystem.DorisInputFile; +import org.apache.doris.filesystem.DorisInputStream; import org.apache.doris.filesystem.DorisOutputFile; import org.apache.doris.filesystem.FileEntry; import org.apache.doris.filesystem.FileIterator; @@ -32,8 +33,10 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -50,6 +53,17 @@ * *

Resolved {@link FileSystem} instances are cached per {@link StorageProperties} reference * (identity-based) to avoid recreating connections on every call. + * + *

Legacy cross-scheme fallback translation. {@link LocationPath} documents a + * compatibility fallback where a path whose scheme does not directly match any configured storage + * is served by an S3-compatible storage named "s3" (e.g. a {@code cos://} path routed to an S3 + * configuration). Concrete filesystems validate URI schemes against their own whitelist, so the + * delegate must not see the foreign scheme. When that fallback fires — and only then — this class + * translates the operand URI to the storage's native scheme before delegating and translates any + * returned paths back to the caller's original scheme, so callers keep comparing paths against + * their metadata (e.g. HMS locations) unchanged. Translation activates only when the rewrite is a + * pure scheme-prefix swap and is therefore exactly reversible; in every other case the original + * URI is passed through untouched. */ public class SpiSwitchingFileSystem implements FileSystem { @@ -82,16 +96,33 @@ public SpiSwitchingFileSystem(FileSystem testDelegate) { /** Resolves the appropriate {@link FileSystem} for the given URI string. */ public FileSystem forPath(String uri) throws IOException { + return resolve(Location.of(uri)).fs; + } + + /** Resolves the appropriate {@link FileSystem} for the given {@link Location}. */ + public FileSystem forLocation(Location location) throws IOException { + return resolve(location).fs; + } + + /** + * Resolves the filesystem for {@code location} and, when the legacy cross-scheme fallback + * fired, the scheme translation to apply on the way in and out. The filesystem choice and + * the translation come from the same {@link LocationPath} resolution — the normalized URI + * is never re-routed. + */ + private Resolved resolve(Location location) throws IOException { if (testDelegate != null) { - return testDelegate; + return new Resolved(testDelegate, null, null); } + String uri = location.uri(); LocationPath lp = LocationPath.of(uri, storagePropertiesMap); StorageProperties sp = lp.getStorageProperties(); if (sp == null) { throw new IOException("No StorageProperties found for path: " + uri); } + FileSystem fs; try { - return cache.computeIfAbsent(sp, props -> { + fs = cache.computeIfAbsent(sp, props -> { try { return FileSystemFactory.getFileSystem(props); } catch (IOException e) { @@ -101,11 +132,48 @@ public FileSystem forPath(String uri) throws IOException { } catch (UncheckedIOException e) { throw e.getCause(); } + String[] prefixes = compatSchemePrefixes(uri, lp, sp); + if (prefixes == null) { + return new Resolved(fs, null, null); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Legacy scheme fallback for path {}: delegating as {}...", uri, prefixes[1]); + } + return new Resolved(fs, prefixes[0], prefixes[1]); } - /** Resolves the appropriate {@link FileSystem} for the given {@link Location}. */ - public FileSystem forLocation(Location location) throws IOException { - return forPath(location.uri()); + /** + * Returns {@code [callerPrefix, delegatePrefix]} (e.g. {@code ["cos://", "s3://"]}) when the + * legacy cross-scheme fallback fired and the normalization is a pure scheme-prefix swap, or + * {@code null} when no translation must happen. The two conditions: + *

    + *
  • the storage type selected by {@link LocationPath} differs from the type the path's + * scheme maps to — i.e. the compatibility fallback in + * {@code LocationPath.findStorageProperties} picked the storage, not a direct match;
    • + *
  • the normalized URI equals the original with only the scheme prefix replaced, so the + * translation is exactly reversible for URIs the delegate returns.
    • + *
+ */ + private static String[] compatSchemePrefixes(String uri, LocationPath lp, StorageProperties sp) { + StorageProperties.Type schemeType = LocationPath.fromSchemaWithContext(uri, lp.getSchema()); + if (schemeType == sp.getType()) { + return null; + } + String normalized = lp.getNormalizedLocation(); + int uriSchemeEnd = uri.indexOf("://"); + int normSchemeEnd = normalized == null ? -1 : normalized.indexOf("://"); + if (uriSchemeEnd < 0 || normSchemeEnd < 0) { + return null; + } + String callerPrefix = uri.substring(0, uriSchemeEnd + 3); + String delegatePrefix = normalized.substring(0, normSchemeEnd + 3); + if (callerPrefix.equalsIgnoreCase(delegatePrefix)) { + return null; + } + if (!uri.substring(uriSchemeEnd + 3).equals(normalized.substring(normSchemeEnd + 3))) { + return null; + } + return new String[] {callerPrefix, delegatePrefix}; } // ----------------------------------------------------------------------- @@ -114,69 +182,119 @@ public FileSystem forLocation(Location location) throws IOException { @Override public boolean exists(Location location) throws IOException { - return forLocation(location).exists(location); + Resolved r = resolve(location); + return r.fs.exists(r.toDelegate(location)); } @Override public void mkdirs(Location location) throws IOException { - forLocation(location).mkdirs(location); + Resolved r = resolve(location); + r.fs.mkdirs(r.toDelegate(location)); } @Override public void delete(Location location, boolean recursive) throws IOException { - forLocation(location).delete(location, recursive); + Resolved r = resolve(location); + r.fs.delete(r.toDelegate(location), recursive); } @Override public void rename(Location src, Location dst) throws IOException { - forLocation(src).rename(src, dst); + Resolved r = resolve(src); + r.fs.rename(r.toDelegate(src), r.toDelegate(dst)); } @Override public FileIterator list(Location location) throws IOException { - return forLocation(location).list(location); + Resolved r = resolve(location); + FileIterator iterator = r.fs.list(r.toDelegate(location)); + if (!r.translated()) { + return iterator; + } + return new FileIterator() { + @Override + public boolean hasNext() throws IOException { + return iterator.hasNext(); + } + + @Override + public FileEntry next() throws IOException { + return r.toCaller(iterator.next()); + } + + @Override + public void close() throws IOException { + iterator.close(); + } + }; } @Override public List listFiles(Location dir) throws IOException { - return forLocation(dir).listFiles(dir); + Resolved r = resolve(dir); + return r.toCaller(r.fs.listFiles(r.toDelegate(dir))); } @Override public List listFilesRecursive(Location dir) throws IOException { - return forLocation(dir).listFilesRecursive(dir); + Resolved r = resolve(dir); + return r.toCaller(r.fs.listFilesRecursive(r.toDelegate(dir))); } @Override public Set listDirectories(Location dir) throws IOException { - return forLocation(dir).listDirectories(dir); + Resolved r = resolve(dir); + Set dirs = r.fs.listDirectories(r.toDelegate(dir)); + if (!r.translated()) { + return dirs; + } + Set mapped = new LinkedHashSet<>(); + for (String d : dirs) { + mapped.add(r.toCaller(d)); + } + return mapped; } @Override public void renameDirectory(Location src, Location dst, Runnable whenSrcNotExists) throws IOException { - forLocation(src).renameDirectory(src, dst, whenSrcNotExists); + Resolved r = resolve(src); + r.fs.renameDirectory(r.toDelegate(src), r.toDelegate(dst), whenSrcNotExists); } @Override public DorisInputFile newInputFile(Location location) throws IOException { - return forLocation(location).newInputFile(location); + Resolved r = resolve(location); + DorisInputFile file = r.fs.newInputFile(r.toDelegate(location)); + return r.translated() ? new TranslatedInputFile(file, location) : file; } @Override public DorisInputFile newInputFile(Location location, long length) throws IOException { - return forLocation(location).newInputFile(location, length); + Resolved r = resolve(location); + DorisInputFile file = r.fs.newInputFile(r.toDelegate(location), length); + return r.translated() ? new TranslatedInputFile(file, location) : file; } @Override public DorisOutputFile newOutputFile(Location location) throws IOException { - return forLocation(location).newOutputFile(location); + Resolved r = resolve(location); + DorisOutputFile file = r.fs.newOutputFile(r.toDelegate(location)); + return r.translated() ? new TranslatedOutputFile(file, location) : file; } @Override public GlobListing globListWithLimit(Location path, String startAfter, long maxBytes, long maxFiles) throws IOException { - return forLocation(path).globListWithLimit(path, startAfter, maxBytes, maxFiles); + Resolved r = resolve(path); + // startAfter and GlobListing's bucket/prefix/maxFile are bucket-relative keys, + // not URIs — only the file entries carry scheme-qualified locations. + GlobListing listing = r.fs.globListWithLimit(r.toDelegate(path), startAfter, maxBytes, maxFiles); + if (!r.translated()) { + return listing; + } + return new GlobListing(r.toCaller(listing.getFiles()), listing.getBucket(), + listing.getPrefix(), listing.getMaxFile()); } @Override @@ -203,4 +321,136 @@ public void close() throws IOException { throw firstError; } } + + /** + * A resolved delegate filesystem plus the (optional) scheme translation to apply. When + * {@code callerPrefix}/{@code delegatePrefix} are null, all methods are identity pass-throughs. + */ + private static final class Resolved { + final FileSystem fs; + /** Scheme prefix as the caller wrote it (e.g. {@code "cos://"}); null when no translation. */ + private final String callerPrefix; + /** Native scheme prefix of the serving storage (e.g. {@code "s3://"}); null when no translation. */ + private final String delegatePrefix; + + Resolved(FileSystem fs, String callerPrefix, String delegatePrefix) { + this.fs = fs; + this.callerPrefix = callerPrefix; + this.delegatePrefix = delegatePrefix; + } + + boolean translated() { + return callerPrefix != null; + } + + /** Caller URI → the scheme the delegate filesystem natively accepts. */ + Location toDelegate(Location location) { + if (!translated()) { + return location; + } + String uri = location.uri(); + if (!uri.regionMatches(true, 0, callerPrefix, 0, callerPrefix.length())) { + return location; + } + return Location.of(delegatePrefix + uri.substring(callerPrefix.length())); + } + + /** Delegate-returned URI → the scheme the caller originally used. */ + String toCaller(String uri) { + if (!translated() || uri == null + || !uri.regionMatches(true, 0, delegatePrefix, 0, delegatePrefix.length())) { + return uri; + } + return callerPrefix + uri.substring(delegatePrefix.length()); + } + + Location toCaller(Location location) { + return translated() ? Location.of(toCaller(location.uri())) : location; + } + + FileEntry toCaller(FileEntry entry) { + if (!translated()) { + return entry; + } + return new FileEntry(toCaller(entry.location()), entry.length(), entry.isDirectory(), + entry.modificationTime(), entry.blocks()); + } + + List toCaller(List entries) { + if (!translated()) { + return entries; + } + List mapped = new ArrayList<>(entries.size()); + for (FileEntry entry : entries) { + mapped.add(toCaller(entry)); + } + return mapped; + } + } + + /** + * Wraps a delegate {@link DorisInputFile} so {@link #location()} reports the caller's + * original URI instead of the scheme-translated one the delegate was opened with — + * callers may feed that location back into this switching filesystem. + */ + private static final class TranslatedInputFile implements DorisInputFile { + private final DorisInputFile delegate; + private final Location callerLocation; + + TranslatedInputFile(DorisInputFile delegate, Location callerLocation) { + this.delegate = delegate; + this.callerLocation = callerLocation; + } + + @Override + public Location location() { + return callerLocation; + } + + @Override + public long length() throws IOException { + return delegate.length(); + } + + @Override + public boolean exists() throws IOException { + return delegate.exists(); + } + + @Override + public long lastModifiedTime() throws IOException { + return delegate.lastModifiedTime(); + } + + @Override + public DorisInputStream newStream() throws IOException { + return delegate.newStream(); + } + } + + /** Output-side counterpart of {@link TranslatedInputFile}. */ + private static final class TranslatedOutputFile implements DorisOutputFile { + private final DorisOutputFile delegate; + private final Location callerLocation; + + TranslatedOutputFile(DorisOutputFile delegate, Location callerLocation) { + this.delegate = delegate; + this.callerLocation = callerLocation; + } + + @Override + public Location location() { + return callerLocation; + } + + @Override + public OutputStream create() throws IOException { + return delegate.create(); + } + + @Override + public OutputStream createOrOverwrite() throws IOException { + return delegate.createOrOverwrite(); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/StoragePropertiesConverter.java b/fe/fe-core/src/main/java/org/apache/doris/fs/StoragePropertiesConverter.java index db6cafdb14d15d..8e44a38dd34eac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/StoragePropertiesConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/StoragePropertiesConverter.java @@ -17,10 +17,12 @@ package org.apache.doris.fs; +import org.apache.doris.common.Config; import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.datasource.property.storage.AzureProperties; import org.apache.doris.datasource.property.storage.BrokerProperties; import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; +import org.apache.doris.datasource.property.storage.OSSHdfsProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.commons.lang3.StringUtils; @@ -82,7 +84,12 @@ public static Map toMap(StorageProperties props) { } map.put("_STORAGE_TYPE_", "AZURE"); } else if (props instanceof HdfsCompatibleProperties) { - map.put("_STORAGE_TYPE_", "HDFS"); + // OSS-HDFS (JindoFS) and plain HDFS share the HDFS-compatible base but are served by + // distinct fe-filesystem providers, so they must carry distinct authoritative markers. + map.put("_STORAGE_TYPE_", props instanceof OSSHdfsProperties ? "OSS_HDFS" : "HDFS"); + // Pass the FE-side hadoop config directory down as a system-injected context key so + // fe-filesystem (which has no fe-core Config) can resolve `hadoop.config.resources`. + map.put("_HADOOP_CONFIG_DIR_", Config.hadoop_config_dir); } else if (props instanceof BrokerProperties) { map.put("_STORAGE_TYPE_", "BROKER"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java index 79a34ad4d382cd..4a8acdce33d8e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java @@ -20,11 +20,15 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.util.HttpURLUtil; import org.apache.doris.common.util.NetUtils; import org.apache.doris.ha.FrontendNodeType; +import org.apache.doris.httpv2.controller.BaseController.ActionAuthorizationInfo; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.httpv2.exception.UnauthorizedException; import org.apache.doris.httpv2.rest.RestBaseController; import org.apache.doris.master.MetaHelper; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.MetaCleaner; import org.apache.doris.persist.Storage; import org.apache.doris.persist.StorageInfo; @@ -55,33 +59,76 @@ public class MetaService extends RestBaseController { private File imageDir = MetaHelper.getMasterImageDir(); - private boolean isFromValidFe(String clientHost, String clientPortStr) { + private Frontend getValidFe(String clientHost, String clientPortStr) { Integer clientPort; try { clientPort = Integer.valueOf(clientPortStr); } catch (Exception e) { LOG.warn("get clientPort error. clientPortStr: {}", clientPortStr, e.getMessage()); - return false; + return null; } Frontend fe = Env.getCurrentEnv().checkFeExist(clientHost, clientPort); if (fe == null) { LOG.warn("request is not from valid FE. client: {}, {}", clientHost, clientPortStr); - return false; } - return true; + return fe; } private void checkFromValidFe(HttpServletRequest request) - throws InvalidClientException { + throws UnauthorizedException { String clientHost = request.getHeader(Env.CLIENT_NODE_HOST_KEY); String clientPort = request.getHeader(Env.CLIENT_NODE_PORT_KEY); - if (!isFromValidFe(clientHost, clientPort)) { - throw new InvalidClientException("invalid client host: " + clientHost + ":" + clientPort - + ", request from " + request.getRemoteHost()); + Frontend fe = getValidFe(clientHost, clientPort); + if (fe == null) { + throw unauthorized(clientHost, clientPort, request); + } + + // If a cluster meta auth token is configured, additionally require the request to + // carry a matching token. An empty token keeps the legacy node-host-only behavior, + // so existing clusters and rolling upgrades are unaffected. + String clusterToken = Config.fe_meta_auth_token; + if (!Strings.isNullOrEmpty(clusterToken)) { + String requestToken = request.getHeader(MetaBaseAction.TOKEN); + if (!clusterToken.equals(requestToken)) { + // Log a masked prefix of both tokens so token-rotation issues are diagnosable + // (e.g. expected "abc***" vs actual "" means the peer sent no token), + // while never revealing the full secret. + LOG.warn("reject meta request with invalid token. client: {}, {}, request from: {}, " + + "expected: {}, actual: {}", + clientHost, clientPort, request.getRemoteAddr(), + maskToken(clusterToken), maskToken(requestToken)); + throw unauthorized(clientHost, clientPort, request); + } } } + private UnauthorizedException unauthorized(String clientHost, String clientPort, HttpServletRequest request) { + return new UnauthorizedException("invalid client host: " + clientHost + ":" + clientPort + + ", request from " + request.getRemoteAddr()); + } + + // Minimum token length required before we reveal a masked prefix in logs. Shorter tokens would + // leak too large a fraction of the secret, so they are hidden entirely with only a length hint. + private static final int MIN_TOKEN_LEN_FOR_PREFIX = 8; + private static final int TOKEN_PREFIX_LEN = 3; + + /** + * Masks a token for logging: reveals only a short leading prefix (e.g. "abc***") so that a + * token mismatch is diagnosable during rotation, while never logging the full secret. Empty + * tokens and tokens too short to safely show a prefix are hidden. + */ + private static String maskToken(String token) { + if (Strings.isNullOrEmpty(token)) { + return ""; + } + if (token.length() < MIN_TOKEN_LEN_FOR_PREFIX) { + // Too short to reveal any prefix without leaking a large fraction of the secret. + return ""; + } + return token.substring(0, TOKEN_PREFIX_LEN) + "***"; + } + @RequestMapping(path = "/image", method = RequestMethod.GET) public Object image(HttpServletRequest request, HttpServletResponse response) { checkFromValidFe(request); @@ -149,6 +196,12 @@ public Object put(HttpServletRequest request, HttpServletResponse response) thro if (port < 0 || port > 65535) { return ResponseEntityBuilder.badRequest("port is invalid. The port number is between 0-65535"); } + // The master pushes image using HttpURLUtil.getHttpPort() (https_port when enable_https=true, + // otherwise http_port), so the expected port must follow the same rule to stay consistent. + int expectedPort = HttpURLUtil.getHttpPort(); + if (port != expectedPort) { + return ResponseEntityBuilder.badRequest("port must be FE HTTP port: " + expectedPort); + } String versionStr = request.getParameter(VERSION); if (Strings.isNullOrEmpty(versionStr)) { @@ -245,9 +298,11 @@ public Object check(HttpServletRequest request, HttpServletResponse response) th @RequestMapping(value = "/dump", method = RequestMethod.GET) public Object dump(HttpServletRequest request, HttpServletResponse response) throws DdlException { - if (Config.enable_all_http_auth) { - executeCheckPassword(request, response); - } + // /dump triggers a full metadata image dump (takes catalog/db/table locks and writes an + // image file), so it must be ADMIN-gated. executeCheckPassword only authenticates the + // caller; enforce the ADMIN privilege explicitly, matching other metadata/debug operations. + ActionAuthorizationInfo authInfo = executeCheckPassword(request, response); + checkGlobalAuth(authInfo.userIdentity, PrivPredicate.ADMIN); /* * Before dump, we acquired the catalog read lock and all databases' read lock and all diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java index e4b2e21c8d7532..12405ef558aa7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java @@ -352,7 +352,7 @@ private Object executeStreamLoad2PC(HttpServletRequest request, String db) { LOG.info("redirect stream load 2PC action to destination={}, db: {}, txn: {}, operation: {}", redirectAddr.toString(), dbName, request.getHeader(TXN_ID_KEY), txnOperation); - RedirectView redirectView = redirectTo(request, redirectAddr); + RedirectView redirectView = redirectToBackend(request, redirectAddr); return redirectView; } catch (Exception e) { @@ -367,7 +367,12 @@ private final synchronized int getLastSelectedBackendIndexAndUpdate() { } private String getCloudClusterName(HttpServletRequest request) { - String cloudClusterName = request.getHeader(SessionVariable.CLOUD_CLUSTER); + String cloudClusterName = request.getHeader(SessionVariable.COMPUTE_GROUP); + if (!Strings.isNullOrEmpty(cloudClusterName)) { + return cloudClusterName; + } + + cloudClusterName = request.getHeader(SessionVariable.CLOUD_CLUSTER); if (!Strings.isNullOrEmpty(cloudClusterName)) { return cloudClusterName; } @@ -669,9 +674,10 @@ private String getAllHeaders(HttpServletRequest request) { private Object createRedirectResponse(HttpServletRequest request, HttpServletResponse response, TNetworkAddress redirectAddr, boolean isStreamLoad, String dbName, String tableName, String label) throws IOException { - String redirectUrl = buildRedirectUrl(request, redirectAddr); + String redirectUrl = buildRedirectUrlToBackend(request, redirectAddr, request.getRequestURI(), + request.getQueryString()); if (!shouldUseBoundedDrainForStreamLoad(isStreamLoad)) { - return redirectTo(request, redirectAddr); + return redirectToBackend(request, redirectAddr); } writeTemporaryRedirect(response, redirectUrl); DrainDecision drainDecision = decideDrainDecisionForStreamLoadRedirect(request); @@ -808,7 +814,7 @@ private RedirectView redirectToStreamLoadForward(HttpServletRequest request, TNe if (!Strings.isNullOrEmpty(queryString)) { redirectQuery = queryString + "&" + redirectQuery; } - String redirectUrl = buildRedirectUrl(request, addr, modifiedPath, redirectQuery); + String redirectUrl = buildRedirectUrlToBackend(request, addr, modifiedPath, redirectQuery); LOG.info("Redirect stream load forward url: {}, forward_to: {}", "http://" + addr.getHostname() + ":" + addr.getPort() + modifiedPath, forwardTarget); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java index 249d8d04c5d57b..b3d3d284a706e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java @@ -100,6 +100,17 @@ protected String buildRedirectUrl(HttpServletRequest request, TNetworkAddress ad protected String buildRedirectUrl(HttpServletRequest request, TNetworkAddress addr, String requestPath, String queryString) { + return buildRedirectUrl(request.getScheme(), request, addr, requestPath, queryString); + } + + // BE's stream-load listener never terminates TLS, so BE-bound redirects must stay "http". + protected String buildRedirectUrlToBackend(HttpServletRequest request, TNetworkAddress addr, + String requestPath, String queryString) { + return buildRedirectUrl("http", request, addr, requestPath, queryString); + } + + private String buildRedirectUrl(String scheme, HttpServletRequest request, TNetworkAddress addr, + String requestPath, String queryString) { String userInfo = null; if (!Strings.isNullOrEmpty(request.getHeader("Authorization"))) { ActionAuthorizationInfo authInfo = getAuthorizationInfo(request); @@ -107,13 +118,13 @@ protected String buildRedirectUrl(HttpServletRequest request, TNetworkAddress ad } try { // Preserve the original request path to avoid re-encoding an already encoded URI path. - URI authorityUri = new URI(request.getScheme(), userInfo, addr.getHostname(), + URI authorityUri = new URI(scheme, userInfo, addr.getHostname(), addr.getPort(), null, null, null); String redirectUrl = authorityUri.toASCIIString() + requestPath; if (!Strings.isNullOrEmpty(queryString)) { redirectUrl += "?" + queryString; } - LOG.info("Redirect url: {}", request.getScheme() + "://" + addr.getHostname() + ":" + LOG.info("Redirect url: {}", scheme + "://" + addr.getHostname() + ":" + addr.getPort() + requestPath); return redirectUrl; } catch (Exception e) { @@ -135,6 +146,15 @@ public RedirectView redirectTo(HttpServletRequest request, TNetworkAddress addr) return redirectView; } + // Use for redirects whose destination is a BE (e.g. stream load), which never speaks HTTPS. + public RedirectView redirectToBackend(HttpServletRequest request, TNetworkAddress addr) { + RedirectView redirectView = new RedirectView( + buildRedirectUrlToBackend(request, addr, request.getRequestURI(), request.getQueryString())); + redirectView.setContentType("text/html;charset=utf-8"); + redirectView.setStatusCode(org.springframework.http.HttpStatus.TEMPORARY_REDIRECT); + return redirectView; + } + public String getRedirectUrL(HttpServletRequest request, TNetworkAddress addr) { return buildRedirectUrl(request, addr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java index 25171e3f411083..43df88ee548506 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java @@ -135,7 +135,9 @@ public static CloseableHttpClient getHttpClient() { } private static String executeRequest(HttpRequestBase request) throws IOException { - try (CloseableHttpClient client = Config.enable_https + // Pick client by this request's own scheme, since this method also serves plain http BE calls. + boolean useHttpsClient = "https".equalsIgnoreCase(request.getURI().getScheme()); + try (CloseableHttpClient client = useHttpsClient ? InternalHttpsUtils.createValidatedHttpClient() : HttpClientBuilder.create().build()) { return client.execute(request, httpResponse -> EntityUtils.toString(httpResponse.getEntity())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java index 3a8dc73bbe2085..fcb3a58a5d5e9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java @@ -259,8 +259,13 @@ private static List getBeList() { */ @RequestMapping(path = "/config", method = RequestMethod.GET) public Object config(HttpServletRequest request, HttpServletResponse response) { - executeCheckPassword(request, response); - checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT); + // This endpoint lists all FE config, matching the SQL "SHOW FRONTEND CONFIG", which + // requires ADMIN. Use an unconditional ADMIN check: checkAdminAuth only enforces the + // privilege when enable_all_http_auth is true, so it would be a no-op by default. + // Sensitive config values (e.g. fe_meta_auth_token) are additionally masked by ConfigBase, + // so they are never returned in plaintext even to an admin. + ActionAuthorizationInfo authInfo = executeCheckPassword(request, response); + checkGlobalAuth(authInfo.userIdentity, PrivPredicate.ADMIN); List> configs = ConfigBase.getConfigInfo(null); // Sort all configs by config key. @@ -320,8 +325,10 @@ public Object config(HttpServletRequest request, HttpServletResponse response) { public Object configurationInfo(HttpServletRequest request, HttpServletResponse response, @RequestParam(value = "type") String type, @RequestBody(required = false) ConfigInfoRequestBody requestBody) { + // Reads FE/BE config via fan-out to the per-node config endpoints, so it must be + // ADMIN-gated too. Unconditional check (see config() above for why checkAdminAuth is not). ActionAuthorizationInfo authInfo = executeCheckPassword(request, response); - checkAdminAuth(authInfo.userIdentity); + checkGlobalAuth(authInfo.userIdentity, PrivPredicate.ADMIN); initHttpExecutor(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ImportAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ImportAction.java index c7252c975ae714..41302d1265ba66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ImportAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/ImportAction.java @@ -93,9 +93,12 @@ public Object fileReview(@RequestBody FileReviewRequestVo body, List fileStatuses = Lists.newArrayList(); try { + // Concrete filesystems only accept their native schemes; normalize legacy compatibility + // schemes (e.g. cos:// with s3.* properties) before crossing the plugin boundary. + String fileUrl = brokerDesc.getFileLocation(fileInfo.getFileUrl()); // get file status try (FileSystem fs = FileSystemFactory.getFileSystem(brokerDesc)) { - for (FileEntry e : fs.listFiles(Location.of(fileInfo.getFileUrl()))) { + for (FileEntry e : fs.listFiles(Location.of(fileUrl))) { fileStatuses.add(new TBrokerFileStatus( e.location().uri(), e.isDirectory(), e.length(), !e.isDirectory())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/common/DataSourceType.java b/fe/fe-core/src/main/java/org/apache/doris/job/common/DataSourceType.java index b188f265957530..3eec77e757b5da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/common/DataSourceType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/common/DataSourceType.java @@ -19,5 +19,6 @@ public enum DataSourceType { MYSQL, - POSTGRES + POSTGRES, + OCEANBASE } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidator.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidator.java index 1c1dc10c0d5051..f75bf03c56e043 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidator.java @@ -17,8 +17,12 @@ package org.apache.doris.job.extensions.insert.streaming; +import org.apache.doris.datasource.jdbc.client.JdbcClient; +import org.apache.doris.datasource.jdbc.client.JdbcClientException; import org.apache.doris.job.cdc.DataSourceConfigKeys; import org.apache.doris.job.common.DataSourceType; +import org.apache.doris.job.exception.JobException; +import org.apache.doris.job.util.StreamingJobUtils; import org.apache.doris.nereids.trees.plans.commands.LoadCommand; import com.fasterxml.jackson.databind.JsonNode; @@ -26,6 +30,9 @@ import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Sets; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; @@ -36,6 +43,7 @@ public class DataSourceConfigValidator { // PostgreSQL unquoted identifier: lowercase letters, digits, underscores, not starting with a digit. private static final Pattern PG_IDENTIFIER_PATTERN = Pattern.compile("^[a-z_][a-z0-9_]*$"); private static final int PG_MAX_IDENTIFIER_LENGTH = 63; + private static final String MYSQL_JDBC_URL_PREFIX = "jdbc:mysql://"; private static final Set ALLOW_SOURCE_KEYS = Sets.newHashSet( DataSourceConfigKeys.JDBC_URL, @@ -58,6 +66,12 @@ public class DataSourceConfigValidator { DataSourceConfigKeys.SERVER_ID ); + private static final Set OCEANBASE_UNSUPPORTED_KEYS = Sets.newHashSet( + DataSourceConfigKeys.SCHEMA, + DataSourceConfigKeys.SLOT_NAME, + DataSourceConfigKeys.PUBLICATION_NAME + ); + private static final Set ALLOW_SSL_MODES = Sets.newHashSet( DataSourceConfigKeys.SSL_MODE_DISABLE, DataSourceConfigKeys.SSL_MODE_REQUIRE, @@ -106,14 +120,72 @@ public static void validateSource(Map input, throw new IllegalArgumentException("Unexpected key: '" + key + "'"); } + if (DataSourceType.OCEANBASE.name().equalsIgnoreCase(dataSourceType) + && OCEANBASE_UNSUPPORTED_KEYS.contains(key)) { + throw new IllegalArgumentException( + "Property '" + key + "' is not supported for OceanBase"); + } + if (!isValidValue(key, value, dataSourceType)) { throw new IllegalArgumentException("Invalid value for key '" + key + "': " + value); } } + validateOceanBaseSource(input, dataSourceType); validateSslVerifyCaPair(input); } + private static void validateOceanBaseSource(Map input, String dataSourceType) { + if (!DataSourceType.OCEANBASE.name().equalsIgnoreCase(dataSourceType)) { + return; + } + if (input.containsKey(DataSourceConfigKeys.JDBC_URL) + && !input.get(DataSourceConfigKeys.JDBC_URL).startsWith(MYSQL_JDBC_URL_PREFIX)) { + throw new IllegalArgumentException( + "OceanBase jdbc_url must start with '" + MYSQL_JDBC_URL_PREFIX + "'"); + } + } + + public static void validateSourceBeforeTableCreation( + DataSourceType sourceType, Map sourceProperties) throws JobException { + if (sourceType == DataSourceType.OCEANBASE) { + validateOceanBaseCompatibilityMode(sourceProperties); + } + } + + private static void validateOceanBaseCompatibilityMode(Map sourceProperties) + throws JobException { + // jdbc:mysql routes through JdbcMySQLClient so Connector/J is initialized consistently. + JdbcClient jdbcClient = StreamingJobUtils.getJdbcClient( + DataSourceType.OCEANBASE, sourceProperties); + try (Connection connection = jdbcClient.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery( + "SHOW VARIABLES LIKE 'ob_compatibility_mode'")) { + if (!resultSet.next()) { + throw new JobException("Failed to determine OceanBase compatibility mode"); + } + String compatibilityMode = resultSet.getString(2); + if ("MYSQL".equalsIgnoreCase(compatibilityMode)) { + return; + } + if ("ORACLE".equalsIgnoreCase(compatibilityMode)) { + throw new JobException( + "OceanBase Oracle compatibility mode is not supported for streaming jobs"); + } + throw new JobException( + "Unsupported OceanBase compatibility mode: " + compatibilityMode); + } catch (JobException e) { + throw e; + } catch (Exception e) { + throw new JobException( + "Failed to validate OceanBase compatibility mode: " + + JdbcClientException.getAllExceptionMessages(e), e); + } finally { + jdbcClient.closeClient(); + } + } + // Cross-field: verify-ca must be paired with a CA cert; otherwise the reader will // silently fall back to the JVM default truststore and likely fail to connect. public static void validateSslVerifyCaPair(Map input) throws IllegalArgumentException { @@ -294,7 +366,7 @@ private static Integer parsePositiveInt(String value) { /** * Check if the offset value is valid for the given data source type. * Supported: initial, snapshot, latest, JSON binlog/lsn position. - * earliest is only supported for MySQL. + * earliest is only supported for MySQL-compatible sources. */ public static boolean isValidOffset(String offset, String dataSourceType) { if (offset == null || offset.isEmpty()) { @@ -305,9 +377,10 @@ public static boolean isValidOffset(String offset, String dataSourceType) { || DataSourceConfigKeys.OFFSET_SNAPSHOT.equalsIgnoreCase(offset)) { return true; } - // earliest only for MySQL + // earliest only for MySQL-compatible sources if (DataSourceConfigKeys.OFFSET_EARLIEST.equalsIgnoreCase(offset)) { - return DataSourceType.MYSQL.name().equalsIgnoreCase(dataSourceType); + return DataSourceType.MYSQL.name().equalsIgnoreCase(dataSourceType) + || DataSourceType.OCEANBASE.name().equalsIgnoreCase(dataSourceType); } if (isJsonOffset(offset)) { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java index b22ccea0b19794..cc7e7e34a18ef3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java @@ -247,9 +247,9 @@ public StreamingInsertJob(String jobName, } /** - * Initialize job from source to database, like multi table mysql to doris. - * 1. get mysql connection info from sourceProperties - * 2. fetch table list from mysql + * Initialize a multi-table job from an external database source to Doris. + * 1. get source connection info from sourceProperties + * 2. fetch the source table list * 3. create doris table if not exists * 4. check whether need full data sync * 5. need => fetch split and write to system table @@ -258,6 +258,8 @@ private void initSourceJob() { try { init(); checkRequiredSourceProperties(); + DataSourceConfigValidator.validateSourceBeforeTableCreation( + dataSourceType, sourceProperties); List createTbls = createTableIfNotExists(); this.syncTables = createTbls; if (sourceProperties.get(DataSourceConfigKeys.INCLUDE_TABLES) == null) { @@ -271,7 +273,7 @@ private void initSourceJob() { this.offsetProvider.initOnCreate(this.syncTables); } catch (Exception ex) { log.warn("init streaming job for {} failed", dataSourceType, ex); - throw new RuntimeException(ex.getMessage()); + throw new RuntimeException(ex.getMessage(), ex); } } @@ -297,11 +299,12 @@ private void checkRequiredSourceProperties() { private List createTableIfNotExists() throws Exception { List syncTbls = new ArrayList<>(); - // Key: source table name (PG/MySQL); Value: CreateTableCommand for the Doris target table. + Map effectiveSourceProperties = buildConvertedSourceProperties(sourceProperties); + // Key: source table name; Value: CreateTableCommand for the Doris target table. // The two names differ when "table..target_table" is configured. LinkedHashMap createTblCmds = StreamingJobUtils.generateCreateTableCmds(targetDb, - dataSourceType, sourceProperties, targetProperties); + dataSourceType, effectiveSourceProperties, targetProperties); Database db = Env.getCurrentEnv().getInternalCatalog().getDbNullable(targetDb); Preconditions.checkNotNull(db, "target database %s does not exist", targetDb); for (Map.Entry entry : createTblCmds.entrySet()) { @@ -310,7 +313,7 @@ private List createTableIfNotExists() throws Exception { if (!db.isTableExist(createTblCmd.getCreateTableInfo().getTableName())) { createTblCmd.run(ConnectContext.get(), null); } - // Use the source (upstream) table name so CDC monitors the correct PG/MySQL table + // Use the upstream table name so CDC monitors the correct source table. syncTbls.add(srcTable); } return syncTbls; @@ -536,7 +539,7 @@ public void alterJob(AlterJobCommand alterJobCommand) throws AnalysisException, Map mergedSourceProperties = new HashMap<>(this.sourceProperties); mergedSourceProperties.putAll(alterJobCommand.getSourceProperties()); Map newConvertedSourceProperties = - StreamingJobUtils.convertCertFile(getDbId(), mergedSourceProperties); + buildConvertedSourceProperties(mergedSourceProperties); this.sourceProperties = mergedSourceProperties; this.convertedSourceProperties = newConvertedSourceProperties; logParts.add("source properties: " + alterJobCommand.getSourceProperties()); @@ -653,10 +656,7 @@ protected AbstractStreamingTask createStreamingTask() throws JobException { return runningStreamTask; } - /** - * for From MySQL TO Database - * @return - */ + /** Create a task for a FROM source TO DATABASE streaming job. */ private AbstractStreamingTask createStreamingMultiTblTask() throws JobException { return new StreamingMultiTblTask(getJobId(), Env.getCurrentEnv().getNextId(), dataSourceType, offsetProvider, getConvertedSourceProperties(), targetDb, targetProperties, jobProperties, @@ -677,11 +677,21 @@ public Backend resolveBoundBackend() throws JobException { private Map getConvertedSourceProperties() throws JobException { if (convertedSourceProperties == null) { - this.convertedSourceProperties = StreamingJobUtils.convertCertFile(getDbId(), sourceProperties); + this.convertedSourceProperties = buildConvertedSourceProperties(sourceProperties); } return convertedSourceProperties; } + private Map buildConvertedSourceProperties(Map inputProperties) + throws JobException { + Map convertedProperties = + StreamingJobUtils.convertCertFile(getDbId(), inputProperties); + convertedProperties.put(DataSourceConfigKeys.JDBC_URL, + StreamingJdbcUrlNormalizer.normalize(dataSourceType, + convertedProperties.get(DataSourceConfigKeys.JDBC_URL))); + return convertedProperties; + } + private Map getOriginTvfProps() { if (originTvfProps == null) { this.originTvfProps = getCurrentTvf().getProperties().getMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJdbcUrlNormalizer.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJdbcUrlNormalizer.java new file mode 100644 index 00000000000000..70f776ab5c1728 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJdbcUrlNormalizer.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.extensions.insert.streaming; + +import org.apache.doris.job.common.DataSourceType; + +import java.util.HashSet; +import java.util.Set; + +/** + * Normalizes JDBC URLs before streaming ingestion uses them for metadata discovery and CDC reads. + * Database-specific rules are kept here so every streaming entry point applies the same read-side + * semantics while leaving unrelated JDBC Catalog write optimizations out of scope. + */ +public final class StreamingJdbcUrlNormalizer { + + private StreamingJdbcUrlNormalizer() { + } + + public static String normalize(DataSourceType sourceType, String jdbcUrl) { + switch (sourceType) { + case MYSQL: + case OCEANBASE: + return normalizeMysql(jdbcUrl); + case POSTGRES: + return jdbcUrl; + default: + throw new IllegalArgumentException("Unsupported data source type: " + sourceType); + } + } + + private static String normalizeMysql(String jdbcUrl) { + String normalizedUrl = jdbcUrl.replace(" ", ""); + Set params = getParams(normalizedUrl); + StringBuilder result = new StringBuilder(normalizedUrl); + setDefaultParam(result, params, "yearIsDateType", "false"); + setDefaultParam(result, params, "tinyInt1isBit", "false"); + setDefaultParam(result, params, "useUnicode", "true"); + setDefaultParam(result, params, "characterEncoding", "utf-8"); + return result.toString(); + } + + private static void setDefaultParam(StringBuilder jdbcUrl, Set params, String param, String value) { + if (params.contains(param)) { + return; + } + char lastChar = jdbcUrl.charAt(jdbcUrl.length() - 1); + if (lastChar != '?' && lastChar != '&') { + jdbcUrl.append(jdbcUrl.indexOf("?") < 0 ? '?' : '&'); + } + jdbcUrl.append(param).append('=').append(value); + } + + private static Set getParams(String jdbcUrl) { + Set params = new HashSet<>(); + int queryIndex = jdbcUrl.indexOf('?'); + if (queryIndex < 0) { + return params; + } + for (String pair : jdbcUrl.substring(queryIndex + 1).split("&")) { + int equalsIndex = pair.indexOf('='); + String name = equalsIndex < 0 ? pair : pair.substring(0, equalsIndex); + if (!name.isEmpty()) { + params.add(name); + } + } + return params; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java index 0a9e8df092133b..13874a35101825 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java @@ -89,7 +89,7 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider { List finishedSplits = new ArrayList<>(); volatile JdbcOffset currentOffset; - Map endBinlogOffset; + volatile Map endBinlogOffset; @SerializedName("chw") // tableID -> splitId -> chunk of highWatermark @@ -117,7 +117,10 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider { /** Cache of Job.syncTables, set by initSplitProgress / replayIfNeed. */ transient List cachedSyncTables; - /** Guards cdcSplitProgress/committedSplitProgress/remainingSplits/finishedSplits. */ + /** + * Guards cdcSplitProgress/committedSplitProgress/remainingSplits/finishedSplits + * and compound binlog updates to currentOffset/endBinlogOffset/hasMoreData. + */ protected final transient Object splitsLock = new Object(); /** @@ -251,9 +254,14 @@ public void updateOffset(Offset offset) { } } } else { - BinlogSplit binlogSplit = (BinlogSplit) newOffset.getSplits().get(0); - binlogOffsetPersist = new HashMap<>(binlogSplit.getStartingOffset()); - binlogOffsetPersist.put(SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID); + synchronized (splitsLock) { + BinlogSplit binlogSplit = (BinlogSplit) newOffset.getSplits().get(0); + binlogOffsetPersist = new HashMap<>(binlogSplit.getStartingOffset()); + binlogOffsetPersist.put(SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID); + currentOffset = newOffset; + hasMoreData = true; + } + return; } this.currentOffset = newOffset; } @@ -286,11 +294,13 @@ public void fetchRemoteMeta(Map properties) throws Exception { } Map newEndOffset = parseCdcResponseData( result.getResponse(), new TypeReference>() {}); - // null→value also counts as a change: upstream may have advanced while fetch was blocked. - if (endBinlogOffset == null || !endBinlogOffset.equals(newEndOffset)) { - hasMoreData = true; + synchronized (splitsLock) { + // null→value also counts as a change: upstream may have advanced while fetch was blocked. + if (endBinlogOffset == null || !endBinlogOffset.equals(newEndOffset)) { + hasMoreData = true; + } + endBinlogOffset = newEndOffset; } - endBinlogOffset = newEndOffset; } catch (TimeoutException te) { log.warn("cdc_client RPC timeout api=/api/fetchEndOffset jobId={} backend={}:{} timeout_sec={}", getJobId(), backend.getHost(), backend.getBrpcPort(), @@ -308,6 +318,8 @@ public boolean hasMoreDataToConsume() { return true; } + JdbcOffset currentOffsetAtCompare; + Map endOffsetAtCompare; synchronized (splitsLock) { if (currentOffset.snapshotSplit()) { if (!remainingSplits.isEmpty()) { @@ -328,19 +340,33 @@ public boolean hasMoreDataToConsume() { if (CollectionUtils.isNotEmpty(remainingSplits)) { return true; } + currentOffsetAtCompare = currentOffset; + endOffsetAtCompare = endBinlogOffset; } - if (MapUtils.isEmpty(endBinlogOffset)) { + if (MapUtils.isEmpty(endOffsetAtCompare)) { return false; } try { - if (!currentOffset.snapshotSplit()) { - BinlogSplit binlogSplit = (BinlogSplit) currentOffset.getSplits().get(0); + if (!currentOffsetAtCompare.snapshotSplit()) { + BinlogSplit binlogSplit = (BinlogSplit) currentOffsetAtCompare.getSplits().get(0); if (MapUtils.isEmpty(binlogSplit.getStartingOffset())) { // snapshot to binlog phase return true; } - hasMoreData = compareOffset(endBinlogOffset, new HashMap<>(binlogSplit.getStartingOffset())); - return hasMoreData; + Map currentBinlogOffset = new HashMap<>(binlogSplit.getStartingOffset()); + int compareResult = compareOffset(endOffsetAtCompare, currentBinlogOffset); + synchronized (splitsLock) { + if (currentOffset != currentOffsetAtCompare || endBinlogOffset != endOffsetAtCompare) { + // The stale result cannot prove that the latest offsets have caught up. + hasMoreData = true; + return true; + } + if (compareResult < 0) { + endBinlogOffset = currentBinlogOffset; + } + hasMoreData = compareResult > 0; + return hasMoreData; + } } else { // snapshot means has data to consume return true; @@ -351,7 +377,7 @@ public boolean hasMoreDataToConsume() { } } - private boolean compareOffset(Map offsetFirst, Map offsetSecond) + protected int compareOffset(Map offsetFirst, Map offsetSecond) throws JobException { Backend backend = StreamingJobUtils.selectBackend(cloudCluster, boundBackendId); CompareOffsetRequest requestParams = @@ -375,7 +401,7 @@ private boolean compareOffset(Map offsetFirst, Map() {}); - return cmp != null && cmp > 0; + return cmp; } catch (TimeoutException te) { log.warn("cdc_client RPC timeout api=/api/compareOffset jobId={} backend={}:{} timeout_sec={}", getJobId(), backend.getHost(), backend.getBrpcPort(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java index 6cbd2e63728db2..0e5bb8fb75319a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java @@ -27,6 +27,7 @@ import org.apache.doris.job.common.DataSourceType; import org.apache.doris.job.exception.JobException; import org.apache.doris.job.extensions.insert.streaming.StreamingInsertJob; +import org.apache.doris.job.extensions.insert.streaming.StreamingJdbcUrlNormalizer; import org.apache.doris.job.offset.Offset; import org.apache.doris.job.util.StreamingJobUtils; import org.apache.doris.nereids.analyzer.UnboundTVFRelation; @@ -105,6 +106,8 @@ public void ensureInitialized(Long jobId, Map originTvfProps) th // Populate default slot/pub into sourceProperties so cleanMeta -> /api/close // carries the resolved names for cdcclient ownership-based cleanup. Map effective = new HashMap<>(originTvfProps); + effective.put(DataSourceConfigKeys.JDBC_URL, StreamingJdbcUrlNormalizer.normalize( + resolvedType, effective.get(DataSourceConfigKeys.JDBC_URL))); StreamingJobUtils.populateDefaultSourceProperties(resolvedType, effective, String.valueOf(jobId)); // Always refresh fields that may be updated via ALTER JOB (e.g. credentials, parallelism). this.sourceProperties = effective; @@ -288,12 +291,17 @@ public void updateOffset(Offset offset) { } } } else { - // Mirror binlog offset into bop so it survives FE checkpoint - BinlogSplit bs = (BinlogSplit) newOffset.getSplits().get(0); - if (MapUtils.isNotEmpty(bs.getStartingOffset())) { - binlogOffsetPersist = new HashMap<>(bs.getStartingOffset()); - binlogOffsetPersist.put(SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID); + synchronized (splitsLock) { + // Mirror binlog offset into bop so it survives FE checkpoint + BinlogSplit bs = (BinlogSplit) newOffset.getSplits().get(0); + if (MapUtils.isNotEmpty(bs.getStartingOffset())) { + binlogOffsetPersist = new HashMap<>(bs.getStartingOffset()); + binlogOffsetPersist.put(SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID); + } + currentOffset = newOffset; + hasMoreData = true; } + return; } this.currentOffset = newOffset; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/util/StreamingJobUtils.java b/fe/fe-core/src/main/java/org/apache/doris/job/util/StreamingJobUtils.java index 033cc4404bb4ff..1ef6bf1809362b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/util/StreamingJobUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/util/StreamingJobUtils.java @@ -359,7 +359,7 @@ public static Map convertCertFile(long dbId, Map *

Returns a {@link LinkedHashMap} whose key is the source (upstream) table name and * whose value is the corresponding {@link CreateTableCommand} that creates the Doris target * table (which may have a different name when {@code table..target_table} is configured). - * Callers must use the map key as the PG/MySQL source table identifier for CDC monitoring and + * Callers must use the map key as the upstream source table identifier for CDC monitoring and * the {@link CreateTableCommand} value for the actual DDL execution. */ public static LinkedHashMap generateCreateTableCmds(String targetDb, @@ -520,14 +520,6 @@ public static List getColumns(JdbcClient jdbcClient, return columns; } - /** - * The remoteDB implementation differs for each data source; - * refer to the hierarchical mapping in the JDBC catalog. - */ - /** - * Populate default resource names into properties, then validate. No-op for sources that - * don't need it. Mutates properties: callers should expect default values to be inserted. - */ public static void resolveAndValidateSource(DataSourceType sourceType, Map properties, String jobId, @@ -591,10 +583,15 @@ public static void populateDefaultSourceProperties(DataSourceType sourceType, } } + /** + * The remoteDB implementation differs for each data source; + * refer to the hierarchical mapping in the JDBC catalog. + */ public static String getRemoteDbName(DataSourceType sourceType, Map properties) { String remoteDb = null; switch (sourceType) { case MYSQL: + case OCEANBASE: remoteDb = properties.get(DataSourceConfigKeys.DATABASE); Preconditions.checkArgument(StringUtils.isNotEmpty(remoteDb), "database is required"); break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java index 1c7d6814a2b31e..49958e46edba60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java @@ -117,9 +117,13 @@ public void addExportJobAndRegisterTask(ExportJob job) throws Exception { try { // delete existing files if (Boolean.parseBoolean(job.getDeleteExistingFiles())) { + // Concrete filesystems only accept their native schemes; normalize legacy + // compatibility schemes (e.g. cos:// with s3.* properties) before crossing + // the plugin boundary. + String exportPath = job.getBrokerDesc().getFileLocation(job.getExportPath()); try (org.apache.doris.filesystem.FileSystem fs = FileSystemFactory.getFileSystem(job.getBrokerDesc())) { - fs.delete(Location.of(FileSystemUtil.extractParentDirectory(job.getExportPath())), true); + fs.delete(Location.of(FileSystemUtil.extractParentDirectory(exportPath)), true); } catch (java.io.IOException e) { throw new UserException("Failed to delete existing files: " + e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java index 534d38a09eed58..73cb7f2c4f47cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java @@ -40,6 +40,8 @@ import org.apache.doris.thrift.TStatusCode; import com.google.common.base.Strings; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -48,10 +50,9 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class GroupCommitManager { @@ -60,10 +61,19 @@ public class GroupCommitManager { private Set blockedTableIds = new HashSet<>(); - // Encoded to BE id map. Only for group commit. - private final Map tableToBeMap = new ConcurrentHashMap<>(); + // Encoded to BE id cache. Only for group commit. + // Bounded so that dropped tables cannot keep their entries alive forever. + private final Cache tableToBeMap = CacheBuilder.newBuilder() + .maximumSize(10000) + .expireAfterAccess(1, TimeUnit.HOURS) + .build(); // Table id to pressure map. Only for group commit. - private final Map tableToPressureMap = new ConcurrentHashMap<>(); + // Bounded like tableToBeMap above: the two hold one entry per group-commit table and are only + // meaningful together, so a dropped table must not keep either of them alive. + private final Cache tableToPressureMap = CacheBuilder.newBuilder() + .maximumSize(10000) + .expireAfterAccess(1, TimeUnit.HOURS) + .build(); public boolean isBlock(long tableId) { return blockedTableIds.contains(tableId); @@ -277,7 +287,7 @@ private long selectBackendForCloudGroupCommitInternal(long tableId, String clust throws DdlException, LoadException { if (LOG.isDebugEnabled()) { LOG.debug("cloud group commit select be info, tableToBeMap {}, tablePressureMap {}", - tableToBeMap.toString(), tableToPressureMap.toString()); + tableToBeMap.asMap().toString(), tableToPressureMap.asMap().toString()); } if (Strings.isNullOrEmpty(cluster)) { ErrorReport.reportDdlException(ErrorCode.ERR_NO_CLUSTER_ERROR); @@ -309,8 +319,8 @@ private long selectBackendForCloudGroupCommitInternal(long tableId, String clust private long selectBackendForLocalGroupCommitInternal(long tableId) throws LoadException { if (LOG.isDebugEnabled()) { - LOG.debug("group commit select be info, tableToBeMap {}, tablePressureMap {}", tableToBeMap.toString(), - tableToPressureMap.toString()); + LOG.debug("group commit select be info, tableToBeMap {}, tablePressureMap {}", + tableToBeMap.asMap().toString(), tableToPressureMap.asMap().toString()); } Long cachedBackendId = getCachedBackend(null, tableId); if (cachedBackendId != null) { @@ -345,26 +355,25 @@ private long selectBackendForLocalGroupCommitInternal(long tableId) throws LoadE @Nullable private Long getCachedBackend(String cluster, long tableId) { OlapTable table = (OlapTable) Env.getCurrentEnv().getInternalCatalog().getTableByTableId(tableId); - if (tableToBeMap.containsKey(encode(cluster, tableId))) { - if (tableToPressureMap.get(tableId) == null) { + String cacheKey = encode(cluster, tableId); + // There are multiple threads getting cached backends for the same table. + // Maybe one thread removes the tableId from the tableToBeMap. + // Another thread gets the same tableId but can not find this tableId. + // So another thread needs to get the random backend. + Long backendId = tableToBeMap.getIfPresent(cacheKey); + if (backendId != null) { + SlidingWindowCounter pressure = tableToPressureMap.getIfPresent(tableId); + if (pressure == null) { return null; - } else if (tableToPressureMap.get(tableId).get() < table.getGroupCommitDataBytes()) { - // There are multiple threads getting cached backends for the same table. - // Maybe one thread removes the tableId from the tableToBeMap. - // Another thread gets the same tableId but can not find this tableId. - // So another thread needs to get the random backend. - Long backendId = tableToBeMap.get(encode(cluster, tableId)); - if (backendId == null) { - return null; - } + } else if (pressure.get() < table.getGroupCommitDataBytes()) { Backend backend = Env.getCurrentSystemInfo().getBackend(backendId); if (isBackendAvailable(backend, cluster)) { return backend.getId(); } else { - tableToBeMap.remove(encode(cluster, tableId)); + tableToBeMap.invalidate(cacheKey); } } else { - tableToBeMap.remove(encode(cluster, tableId)); + tableToBeMap.invalidate(cacheKey); } } return null; @@ -426,11 +435,12 @@ public void updateLoadData(long tableId, long receiveData) { } private void updateLoadDataInternal(long tableId, long receiveData) { - if (tableToPressureMap.containsKey(tableId)) { - tableToPressureMap.get(tableId).add(receiveData); + SlidingWindowCounter pressure = tableToPressureMap.getIfPresent(tableId); + if (pressure != null) { + pressure.add(receiveData); if (LOG.isDebugEnabled()) { LOG.debug("Update load data for table {}, receiveData {}, tablePressureMap {}", tableId, receiveData, - tableToPressureMap.toString()); + tableToPressureMap.asMap().toString()); } } else if (LOG.isDebugEnabled()) { LOG.debug("can not find table id {}", tableId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java index e4971a96b7bbc3..60249b8df51979 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java @@ -137,11 +137,19 @@ public void setCloudCluster() throws UserException { if (!request.isSetToken() && !request.isSetAuthCode() && !Strings.isNullOrEmpty(userName)) { ctx.setCurrentUserIdentity(resolveCloudLoadUserIdentity(userName)); } - if ((request.isSetToken() || request.isSetAuthCode()) && request.isSetBackendId()) { + if (request.isSetBackendId()) { long backendId = request.getBackendId(); Backend backend = Env.getCurrentSystemInfo().getBackend(backendId); Preconditions.checkNotNull(backend); - ctx.setCloudCluster(backend.getCloudClusterName()); + String computeGroup = backend.getCloudClusterName(); + // Token/auth-code and user-less internal loads keep their existing trusted path. Regular + // stream loads must still validate compute group privilege, existence, and status. + if (request.isSetToken() || request.isSetAuthCode() || Strings.isNullOrEmpty(userName)) { + ctx.setCloudCluster(computeGroup); + } else { + ((CloudEnv) Env.getCurrentEnv()).changeCloudCluster(computeGroup, ctx); + } + request.setCloudCluster(computeGroup); return; } if (!Strings.isNullOrEmpty(request.getCloudCluster())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java index a38273ad540eeb..fbc34895a43ca7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java @@ -325,6 +325,20 @@ protected void analyzeCommand(LoadCommand command, Database db, ConnectContext c for (NereidsDataDescription dataDescription : command.getDataDescriptions()) { dataDescription.analyzeWithoutCheckPriv(db.getFullName()); } + // This replay path re-parses originStmt and skips LoadCommand.run(), which is where file + // paths are normalized on fresh submission. Concrete filesystems only accept their native + // schemes, so re-apply the same normalization (e.g. cos:// with s3.* properties -> s3://) + // before the paths land in BrokerFileGroup, or a pending job rescheduled after an FE + // restart / master failover would fail scheme validation and be cancelled. + BrokerDesc brokerDesc = command.getBrokerDesc(); + if (brokerDesc != null && !brokerDesc.isMultiLoadBroker()) { + for (NereidsDataDescription dataDescription : command.getDataDescriptions()) { + List filePaths = dataDescription.getFilePaths(); + for (int i = 0; i < filePaths.size(); i++) { + filePaths.set(i, brokerDesc.getFileLocation(filePaths.get(i))); + } + } + } checkAndSetDataSourceInfoByNereids(db, command.getDataDescriptions(), ctx); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RLTaskTxnCommitAttachment.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RLTaskTxnCommitAttachment.java index dd9c4ed85ce908..48607266075ff1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RLTaskTxnCommitAttachment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RLTaskTxnCommitAttachment.java @@ -18,6 +18,7 @@ package org.apache.doris.load.routineload; import org.apache.doris.cloud.proto.Cloud.RLTaskTxnCommitAttachmentPB; +import org.apache.doris.common.Config; import org.apache.doris.load.routineload.kafka.KafkaProgress; import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.thrift.TRLTaskTxnCommitAttachment; @@ -26,6 +27,7 @@ import org.apache.doris.transaction.TxnCommitAttachment; import com.google.gson.annotations.SerializedName; +import org.apache.commons.lang3.StringUtils; // {"progress": "", "backendId": "", "taskSignature": "", "numOfErrorData": "", // "numOfTotalData": "", "taskId": "", "jobId": ""} @@ -46,6 +48,7 @@ public class RLTaskTxnCommitAttachment extends TxnCommitAttachment { @SerializedName(value = "pro") private RoutineLoadProgress progress; private String errorLogUrl; + private String firstErrorMsg; public RLTaskTxnCommitAttachment() { super(TransactionState.LoadJobSourceType.ROUTINE_LOAD_TASK); @@ -75,6 +78,9 @@ public RLTaskTxnCommitAttachment(TRLTaskTxnCommitAttachment rlTaskTxnCommitAttac if (rlTaskTxnCommitAttachment.isSetErrorLogUrl()) { this.errorLogUrl = rlTaskTxnCommitAttachment.getErrorLogUrl(); } + if (rlTaskTxnCommitAttachment.isSetFirstErrorMsg()) { + this.firstErrorMsg = abbreviateFirstErrorMsg(rlTaskTxnCommitAttachment.getFirstErrorMsg()); + } } public RLTaskTxnCommitAttachment(RLTaskTxnCommitAttachmentPB rlTaskTxnCommitAttachment) { @@ -92,6 +98,11 @@ public RLTaskTxnCommitAttachment(RLTaskTxnCommitAttachmentPB rlTaskTxnCommitAtta this.progress = progress; this.errorLogUrl = rlTaskTxnCommitAttachment.getErrorLogUrl(); + this.firstErrorMsg = abbreviateFirstErrorMsg(rlTaskTxnCommitAttachment.getFirstErrorMsg()); + } + + private static String abbreviateFirstErrorMsg(String firstErrorMsg) { + return StringUtils.abbreviate(firstErrorMsg, Config.first_error_msg_max_length); } public long getJobId() { @@ -134,6 +145,10 @@ public String getErrorLogUrl() { return errorLogUrl; } + public String getFirstErrorMsg() { + return firstErrorMsg; + } + @Override public String toString() { return "RLTaskTxnCommitAttachment [filteredRows=" + filteredRows diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 3223ff913594e1..9873368f405114 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -274,8 +274,10 @@ public boolean isFinalState() { protected Expr deleteCondition; // TODO(ml): error sample - // save the latest 3 error log urls - private Queue errorLogUrls = EvictingQueue.create(3); + // Save the latest 3 error log URLs in memory. The corresponding first error message + // uses the same lifecycle and should not be persisted with the job. + private transient Queue errorLogUrls = EvictingQueue.create(3); + private transient String firstErrorMsg = ""; @SerializedName("ccid") private String cloudClusterId; @@ -809,6 +811,10 @@ public Queue getErrorLogUrls() { return errorLogUrls; } + public String getFirstErrorMsg() { + return Strings.nullToEmpty(firstErrorMsg); + } + // RoutineLoadScheduler will run this method at fixed interval, and renew the timeout tasks public void processTimeoutTasks() { writeLock(); @@ -954,10 +960,7 @@ private void updateNumOfData(long numOfTotalRows, long numOfErrorRows, long unse + "when current total rows is more than base or the filter ratio is more than the max") .build()); } - // reset currentTotalNum, currentErrorNum and otherMsg - this.jobStatistic.currentErrorRows = 0; - this.jobStatistic.currentTotalRows = 0; - this.otherMsg = ""; + resetCurrentErrorStatistics(); this.jobStatistic.currentAbortedTaskNum = 0; } else if (this.jobStatistic.currentErrorRows > maxErrorNum || (this.jobStatistic.currentTotalRows > 0 @@ -977,13 +980,18 @@ private void updateNumOfData(long numOfTotalRows, long numOfErrorRows, long unse "current error rows is more than max_error_number " + "or the max_filter_ratio is more than the value set"), isReplay); } - // reset currentTotalNum, currentErrorNum and otherMsg - this.jobStatistic.currentErrorRows = 0; - this.jobStatistic.currentTotalRows = 0; - this.otherMsg = ""; + resetCurrentErrorStatistics(); } } + private void resetCurrentErrorStatistics() { + this.jobStatistic.currentErrorRows = 0; + this.jobStatistic.currentTotalRows = 0; + this.otherMsg = ""; + this.errorLogUrls.clear(); + this.firstErrorMsg = ""; + } + protected void replayUpdateProgress(RLTaskTxnCommitAttachment attachment) { try { updateNumOfData(attachment.getTotalRows(), attachment.getFilteredRows(), attachment.getUnselectedRows(), @@ -1408,8 +1416,10 @@ private void executeTaskOnTxnStatusChanged(RoutineLoadTaskInfo routineLoadTaskIn routineLoadTaskInfo.handleTaskByTxnCommitAttachment(rlTaskTxnCommitAttachment); } - if (rlTaskTxnCommitAttachment != null && !Strings.isNullOrEmpty(rlTaskTxnCommitAttachment.getErrorLogUrl())) { + if (rlTaskTxnCommitAttachment != null + && !Strings.isNullOrEmpty(rlTaskTxnCommitAttachment.getErrorLogUrl())) { errorLogUrls.add(rlTaskTxnCommitAttachment.getErrorLogUrl()); + firstErrorMsg = Strings.nullToEmpty(rlTaskTxnCommitAttachment.getFirstErrorMsg()); } routineLoadTaskInfo.setTxnStatus(txnStatus); @@ -1712,6 +1722,7 @@ public List getShowInfo() { row.add(userIdentity.getQualifiedUser()); row.add(comment); row.add(getClusterInfo()); + row.add(getFirstErrorMsg()); return row; } finally { readUnlock(); @@ -1951,6 +1962,8 @@ public void write(DataOutput out) throws IOException { @Override public void gsonPostProcess() throws IOException { + errorLogUrls = EvictingQueue.create(3); + firstErrorMsg = ""; if (tableId == 0) { isMultiTable = true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java b/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java index 5acec62ae59a77..73fceec8b39481 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java @@ -149,7 +149,7 @@ private static void checkFile(File file) throws IOException { public static ResponseBody doGet(String url, int timeout, Class clazz) throws IOException { Map headers = HttpURLUtil.getNodeIdentHeaders(); - LOG.info("meta helper, url: {}, timeout: {}, headers: {}", url, timeout, headers); + LOG.info("meta helper, url: {}, timeout: {}, header names: {}", url, timeout, headers.keySet()); String response = HttpUtils.doGet(url, headers, timeout); try { return parseResponse(response, clazz); diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java b/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java index b88780a9f4c01d..7649b3b1c6780a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java @@ -58,13 +58,16 @@ public class CloudMetrics { // Per-method meta-service RPC metrics public static AutoMappedMetric META_SERVICE_RPC_TOTAL; public static AutoMappedMetric META_SERVICE_RPC_FAILED; + public static AutoMappedMetric META_SERVICE_RPC_RATE_LIMITED; public static AutoMappedMetric META_SERVICE_RPC_RETRY; public static AutoMappedMetric> META_SERVICE_RPC_PER_SECOND; public static AutoMappedMetric META_SERVICE_RPC_LATENCY; + public static AutoMappedMetric META_SERVICE_RPC_RATE_LIMIT_WAIT_LATENCY; // Aggregate meta-service metrics public static LongCounterMetric META_SERVICE_RPC_ALL_TOTAL; public static LongCounterMetric META_SERVICE_RPC_ALL_FAILED; + public static LongCounterMetric META_SERVICE_RPC_ALL_RATE_LIMITED; public static LongCounterMetric META_SERVICE_RPC_ALL_RETRY; public static GaugeMetricImpl META_SERVICE_RPC_ALL_PER_SECOND; @@ -155,6 +158,9 @@ protected static void init() { META_SERVICE_RPC_FAILED = MetricRepo.addLabeledMetrics("method", () -> new LongCounterMetric("meta_service_rpc_failed", MetricUnit.NOUNIT, "failed meta service RPC calls")); + META_SERVICE_RPC_RATE_LIMITED = MetricRepo.addLabeledMetrics("method", () -> + new LongCounterMetric("meta_service_rpc_rate_limited", MetricUnit.NOUNIT, + "rate limited meta service RPC calls")); META_SERVICE_RPC_RETRY = MetricRepo.addLabeledMetrics("method", () -> new LongCounterMetric("meta_service_rpc_retry", MetricUnit.NOUNIT, "meta service RPC retry attempts")); @@ -170,6 +176,14 @@ protected static void init() { }); MetricRepo.DORIS_METRIC_REGISTER.addHistogramMetrics( "meta_service_rpc_latency", META_SERVICE_RPC_LATENCY, Config::isCloudMode); + META_SERVICE_RPC_RATE_LIMIT_WAIT_LATENCY = new AutoMappedMetric<>(methodName -> { + List labels = Collections.singletonList(new MetricLabel("method", methodName)); + return new HistogramMetric(MetricRegistry.name("meta_service", "rpc", "rate_limit_wait", + "latency", "ms"), labels); + }); + MetricRepo.DORIS_METRIC_REGISTER.addHistogramMetrics( + "meta_service_rpc_rate_limit_wait_latency", META_SERVICE_RPC_RATE_LIMIT_WAIT_LATENCY, + Config::isCloudMode); // Aggregate meta-service metrics META_SERVICE_RPC_ALL_TOTAL = new LongCounterMetric("meta_service_rpc_all_total", @@ -178,6 +192,9 @@ protected static void init() { META_SERVICE_RPC_ALL_FAILED = new LongCounterMetric("meta_service_rpc_all_failed", MetricUnit.NOUNIT, "total failed meta service RPC calls"); MetricRepo.DORIS_METRIC_REGISTER.addMetrics(META_SERVICE_RPC_ALL_FAILED); + META_SERVICE_RPC_ALL_RATE_LIMITED = new LongCounterMetric("meta_service_rpc_all_rate_limited", + MetricUnit.NOUNIT, "total rate limited meta service RPC calls"); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(META_SERVICE_RPC_ALL_RATE_LIMITED); META_SERVICE_RPC_ALL_RETRY = new LongCounterMetric("meta_service_rpc_all_retry", MetricUnit.NOUNIT, "total meta service RPC retry attempts"); MetricRepo.DORIS_METRIC_REGISTER.addMetrics(META_SERVICE_RPC_ALL_RETRY); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerDorisAccessControllerFactory.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerDorisAccessControllerFactory.java index 28093ad7886ee7..7e23769911520c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerDorisAccessControllerFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerDorisAccessControllerFactory.java @@ -22,6 +22,11 @@ import java.util.Map; public class RangerDorisAccessControllerFactory implements AccessControllerFactory { + private static class SingletonHolder { + // Every controller starts a Ranger policy refresher, so all Env instances must share one controller. + private static final RangerDorisAccessController INSTANCE = new RangerDorisAccessController("doris"); + } + @Override public String factoryIdentifier() { return "ranger-doris"; @@ -29,6 +34,6 @@ public String factoryIdentifier() { @Override public RangerDorisAccessController createAccessController(Map prop) { - return new RangerDorisAccessController("doris"); + return SingletonHolder.INSTANCE; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 01d9139577578b..f42112d1e11d28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -850,6 +850,11 @@ private PlanFragment getPlanFragmentForPhysicalFileScan(PhysicalFileScan fileSca scanNode.setNereidsId(fileScan.getId()); context.getNereidsIdToPlanNodeIdMap().put(fileScan.getId(), scanNode.getId()); scanNode.setPushDownAggNoGrouping(context.getRelationPushAggOp(fileScan.getRelationId())); + scanNode.setPushDownCountSlotIds(context.getRelationPushCountArgumentExprIds(fileScan.getRelationId()) + .stream() + .map(exprId -> Objects.requireNonNull(context.findSlotRef(exprId), + "missing slot for pushed-down COUNT argument " + exprId).getSlotId()) + .collect(Collectors.toList())); scanNode.setHasPartitionPredicate(fileScan.hasPartitionPredicate()); if (fileScan.getStats() != null) { @@ -1401,6 +1406,10 @@ public PlanFragment visitPhysicalStorageLayerAggregate( context.setRelationPushAggOp( storageLayerAggregate.getRelation().getRelationId(), pushAggOp); + context.setRelationPushCountArgumentExprIds( + storageLayerAggregate.getRelation().getRelationId(), + pushAggOp == TPushAggOp.COUNT + ? storageLayerAggregate.getCountArgumentExprIds() : ImmutableList.of()); PlanFragment planFragment = storageLayerAggregate.getRelation().accept(this, context); @@ -2487,14 +2496,34 @@ public PlanFragment visitPhysicalSetOperation( setOperationNode.setColocate(true); } - // TODO: open comment when support `enable_local_shuffle_planner` - // for (Plan child : setOperation.children()) { - // PhysicalPlan childPhysicalPlan = (PhysicalPlan) child; - // if (JoinUtils.isStorageBucketed(childPhysicalPlan.getPhysicalProperties())) { - // setOperationNode.setDistributionMode(DistributionMode.BUCKET_SHUFFLE); - // break; - // } - // } + // Storage-bucketed children only appear when the FE local shuffle planner is active: + // ChildrenPropertiesRegulator and RequestPropertyDeriver both gate the bucket-shuffle + // alternative on enableLocalShufflePlanner. Gate the marker on the same flag so the + // dependency is explicit and a future planner change that produced a STORAGE_BUCKETED + // distribution outside the local-shuffle planner cannot silently mark BUCKET_SHUFFLE here. + // + // Within that gate a storage-bucketed child means the regulator chose the bucket shuffle + // alternative (it enforces the other children onto the basic child's buckets), so the marker + // simply follows that decision. It must not re-check the table id independently: the basic + // child selection in the regulator is the single place that vets the layout, and re-checking + // here could suppress a bucket shuffle the property model already committed to and desync a + // parent that aligned to the set operation output. + // + // Unlike hash join, BUCKET_SHUFFLE is not exclusive with isColocate above: for a set + // operation isColocate describes the bucket-aligned scheduling of the fragment (the + // basic child scans buckets directly), while BUCKET_SHUFFLE describes how the other + // children arrive (bucket-shuffle exchanges). Both routes converge to the same + // bucket-hash local exchange requirement in SetOperationNode.enforceAndDeriveLocalExchange. + if (context.getSessionVariable() != null + && context.getSessionVariable().isEnableLocalShufflePlanner()) { + for (Plan child : setOperation.children()) { + PhysicalPlan childPhysicalPlan = (PhysicalPlan) child; + if (JoinUtils.isStorageBucketed(childPhysicalPlan.getPhysicalProperties())) { + setOperationNode.setDistributionMode(DistributionMode.BUCKET_SHUFFLE); + break; + } + } + } return setOperationFragment; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index f680936b5e5ef4..936f4cbe1a9235 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -52,6 +52,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -106,6 +107,7 @@ public class PlanTranslatorContext { private final Map cteScanNodeMap = Maps.newHashMap(); private final Map tablePushAggOp = Maps.newHashMap(); + private final Map> tablePushCountArgumentExprIds = Maps.newHashMap(); private final Map> statsUnknownColumnsMap = Maps.newHashMap(); @@ -430,6 +432,14 @@ public TPushAggOp getRelationPushAggOp(RelationId relationId) { return tablePushAggOp.getOrDefault(relationId, TPushAggOp.NONE); } + public void setRelationPushCountArgumentExprIds(RelationId relationId, List exprIds) { + tablePushCountArgumentExprIds.put(relationId, Lists.newArrayList(exprIds)); + } + + public List getRelationPushCountArgumentExprIds(RelationId relationId) { + return tablePushCountArgumentExprIds.getOrDefault(relationId, Collections.emptyList()); + } + public boolean isTopMaterializeNode() { return isTopMaterializeNode; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java index b1dd098324eefe..00c61916329e68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanNode; @@ -79,6 +80,9 @@ static Classification classify(TRuntimeFilterType filterType, Expr targetExpr, if (hasUnsupportedAutomaticPartitionExpression(partitionInfo)) { return Classification.unsupported("automatic partition expression boundary is not modeled"); } + if (nereidsTargetExpr.containsType(NoneMovableFunction.class)) { + return Classification.unsupported("target expression contains non-movable function"); + } if (targetExpr instanceof SlotRef) { SlotRef slotRef = (SlotRef) targetExpr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java index 45df90d3d1d091..7f67311204d349 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java @@ -416,20 +416,13 @@ public Pair getJoinConstraint(Long joinTableBitmap, Lon continue; } - if (joinConstraint.getJoinType().isSemiOrAntiJoin()) { - if (!LongBitmap.isSubset(joinConstraint.getMinLeftHand(), joinTableBitmap) - || !LongBitmap.isSubset(joinConstraint.getMinRightHand(), joinTableBitmap)) { + if (joinConstraint.getJoinType().isSemiJoin()) { + if (LongBitmap.isSubset(joinConstraint.getRightHand(), leftTableBitmap) + && !LongBitmap.isSubset(joinConstraint.getRightHand(), leftTableBitmap)) { continue; } - - Long constrainedSide = joinConstraint.getJoinType().isRightSemiOrAntiJoin() - ? joinConstraint.getLeftHand() : joinConstraint.getRightHand(); - if (LongBitmap.isOverlap(constrainedSide, leftTableBitmap) - && !constrainedSide.equals(leftTableBitmap)) { - continue; - } - if (LongBitmap.isOverlap(constrainedSide, rightTableBitmap) - && !constrainedSide.equals(rightTableBitmap)) { + if (LongBitmap.isSubset(joinConstraint.getRightHand(), rightTableBitmap) + && !joinConstraint.getRightHand().equals(rightTableBitmap)) { continue; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c470bc789d9e78..42695f85a8d34f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.analysis.ColumnNullableType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.analysis.DbName; import org.apache.doris.analysis.EncryptKeyName; import org.apache.doris.analysis.PassVar; @@ -140,6 +141,7 @@ import org.apache.doris.nereids.DorisParser.CleanLabelContext; import org.apache.doris.nereids.DorisParser.CollateContext; import org.apache.doris.nereids.DorisParser.ColumnDefContext; +import org.apache.doris.nereids.DorisParser.ColumnDefWithPathContext; import org.apache.doris.nereids.DorisParser.ColumnDefsContext; import org.apache.doris.nereids.DorisParser.ColumnReferenceContext; import org.apache.doris.nereids.DorisParser.CommentRelationHintContext; @@ -1094,6 +1096,7 @@ import org.apache.doris.nereids.types.coercion.CharacterType; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.RelationUtil; +import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.policy.FilterType; import org.apache.doris.policy.PolicyTypeEnum; @@ -1154,6 +1157,16 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { private static String DEFAULT_NESTED_COLUMN_NAME = "unnest"; private static String DEFAULT_ORDINALITY_COLUMN_NAME = "ordinality"; + private static class ColumnDefinitionWithPath { + private final ColumnDefinition columnDefinition; + private final ColumnPath columnPath; + + private ColumnDefinitionWithPath(ColumnDefinition columnDefinition, ColumnPath columnPath) { + this.columnDefinition = columnDefinition; + this.columnPath = columnPath; + } + } + // Sort the parameters with token position to keep the order with original placeholders // in prepared statement.Otherwise, the order maybe broken private final Map tokenPosToParameters = Maps.newTreeMap((pos1, pos2) -> { @@ -1174,6 +1187,18 @@ public LogicalPlanBuilder(Map selectHintMap) { this.selectHintMap = selectHintMap; } + private static String requireNonEmptyColumnIdentifier(ParserRuleContext ctx, String identifier) { + if (identifier.isEmpty()) { + throw new ParseException("Quoted identifier cannot be empty", ctx); + } + return identifier; + } + + private static ColumnPath parseColumnPath(ParserRuleContext ctx, List parts) { + parts.forEach(part -> requireNonEmptyColumnIdentifier(ctx, part)); + return ColumnPath.of(parts); + } + @SuppressWarnings("unchecked") protected T typedVisit(ParseTree ctx) { return (T) ctx.accept(this); @@ -3728,18 +3753,7 @@ public Literal visitIntegerLiteral(IntegerLiteralContext ctx) { @Override public Literal visitStringLiteral(StringLiteralContext ctx) { - String txt = ctx.STRING_LITERAL().getText(); - String s = txt.substring(1, txt.length() - 1); - if (txt.charAt(0) == '\'') { - // for single quote string, '' should be converted to ' - s = s.replace("''", "'"); - } else if (txt.charAt(0) == '"') { - // for double quote string, "" should be converted to " - s = s.replace("\"\"", "\""); - } - if (!SqlModeHelper.hasNoBackSlashEscapes()) { - s = LogicalPlanBuilderAssistant.escapeBackSlash(s); - } + String s = SqlLiteralUtils.parseStringLiteral(ctx.STRING_LITERAL().getText()); int strLength = Utils.containChinese(s) ? s.length() * StringLikeLiteral.CHINESE_CHAR_BYTE_LENGTH : s.length(); if (strLength > ScalarType.MAX_VARCHAR_LENGTH) { return new StringLiteral(s); @@ -3906,6 +3920,13 @@ public List visitIdentifierSeq(IdentifierSeqContext ctx) { .collect(ImmutableList.toImmutableList()); } + @Override + public List visitQualifiedName(QualifiedNameContext ctx) { + return ctx.identifier().stream() + .map(RuleContext::getText) + .collect(ImmutableList.toImmutableList()); + } + @Override public EqualTo visitUpdateAssignment(UpdateAssignmentContext ctx) { return new EqualTo(new UnboundSlot( @@ -4211,9 +4232,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { e.getCause()); } } - //comment should remove '\' and '(") at the beginning and end - String comment = ctx.comment != null ? ctx.comment.getText().substring(1, ctx.comment.getText().length() - 1) - .replace("\\", "") : ""; + String comment = ctx.comment != null + ? SqlLiteralUtils.parseStringLiteral(ctx.comment.getText()) : ""; long autoIncInitValue = -1; if (ctx.AUTO_INCREMENT() != null) { if (ctx.autoIncInitValue != null) { @@ -4231,7 +4251,65 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { ? Optional.of(new GeneratedColumnDesc(ctx.generatedExpr.getText(), getExpression(ctx.generatedExpr))) : Optional.empty(); return new ColumnDefinition(colName, colType, isKey, aggType, nullableType, autoIncInitValue, defaultValue, - onUpdateDefaultValue, comment, desc); + onUpdateDefaultValue, comment, ctx.comment != null, true, desc); + } + + @Override + public ColumnDefinitionWithPath visitColumnDefWithPath(ColumnDefWithPathContext ctx) { + if (ctx.columnDef() != null) { + ColumnDefinition columnDefinition = visitColumnDef(ctx.columnDef()); + ColumnPath columnPath = parseColumnPath(ctx, Collections.singletonList(columnDefinition.getName())); + return new ColumnDefinitionWithPath(columnDefinition, columnPath); + } + + ColumnPath columnPath = parseColumnPath(ctx, ctx.colNames.stream() + .map(RuleContext::getText) + .collect(Collectors.toList())); + String colName = columnPath.getLeafName(); + DataType colType = ctx.type instanceof PrimitiveDataTypeContext + ? visitPrimitiveDataType(((PrimitiveDataTypeContext) ctx.type)) + : ctx.type instanceof ComplexDataTypeContext + ? visitComplexDataType((ComplexDataTypeContext) ctx.type) + : ctx.type instanceof VariantPredefinedFieldsContext + ? visitVariantPredefinedFields((VariantPredefinedFieldsContext) ctx.type) + : visitAggStateDataType((AggStateDataTypeContext) ctx.type); + colType = colType.conversion(); + boolean isKey = ctx.KEY() != null; + ColumnNullableType nullableType = ColumnNullableType.DEFAULT; + if (ctx.NOT() != null) { + nullableType = ColumnNullableType.NOT_NULLABLE; + } else if (ctx.nullable != null) { + nullableType = ColumnNullableType.NULLABLE; + } + String aggTypeString = ctx.aggType != null ? ctx.aggType.getText() : null; + AggregateType aggType = null; + if (aggTypeString != null) { + try { + aggType = AggregateType.valueOf(aggTypeString.toUpperCase()); + } catch (Exception e) { + throw new AnalysisException(String.format("Aggregate type %s is unsupported", aggTypeString), + e.getCause()); + } + } + String comment = ctx.comment != null + ? SqlLiteralUtils.parseStringLiteral(ctx.comment.getText()) : ""; + long autoIncInitValue = -1; + if (ctx.AUTO_INCREMENT() != null) { + if (ctx.autoIncInitValue != null) { + autoIncInitValue = Long.valueOf(ctx.autoIncInitValue.getText()); + if (autoIncInitValue < 0) { + throw new AnalysisException("AUTO_INCREMENT start value can not be negative."); + } + } else { + autoIncInitValue = Long.valueOf(1); + } + } + Optional desc = ctx.generatedExpr != null + ? Optional.of(new GeneratedColumnDesc(ctx.generatedExpr.getText(), getExpression(ctx.generatedExpr))) + : Optional.empty(); + ColumnDefinition columnDefinition = new ColumnDefinition(colName, colType, isKey, aggType, nullableType, + autoIncInitValue, Optional.empty(), Optional.empty(), comment, ctx.comment != null, true, desc); + return new ColumnDefinitionWithPath(columnDefinition, columnPath); } @Override @@ -5417,14 +5495,11 @@ public List visitComplexColTypeList(ComplexColTypeListContext ctx) @Override public StructField visitComplexColType(ComplexColTypeContext ctx) { - String comment; - if (ctx.commentSpec() != null) { - comment = ctx.commentSpec().STRING_LITERAL().getText(); - comment = LogicalPlanBuilderAssistant.escapeBackSlash(comment.substring(1, comment.length() - 1)); - } else { - comment = ""; - } - return new StructField(ctx.identifier().getText(), typedVisit(ctx.dataType()), true, comment); + String comment = ctx.commentSpec() == null ? "" + : SqlLiteralUtils.parseStringLiteral( + ctx.commentSpec().STRING_LITERAL().getText()); + return new StructField(ctx.identifier().getText(), typedVisit(ctx.dataType()), true, + comment, ctx.commentSpec() != null); } private String parseConstant(ConstantContext context) { @@ -6105,7 +6180,7 @@ public AlterTableCommand visitAlterTableProperties(DorisParser.AlterTablePropert @Override public AlterTableOp visitAddColumnClause(AddColumnClauseContext ctx) { - ColumnDefinition columnDefinition = visitColumnDef(ctx.columnDef()); + ColumnDefinitionWithPath columnDefinitionWithPath = visitColumnDefWithPath(ctx.columnDefWithPath()); ColumnPosition columnPosition = null; if (ctx.columnPosition() != null) { if (ctx.columnPosition().FIRST() != null) { @@ -6118,7 +6193,8 @@ public AlterTableOp visitAddColumnClause(AddColumnClauseContext ctx) { Map properties = ctx.properties != null ? Maps.newHashMap(visitPropertyClause(ctx.properties)) : Maps.newHashMap(); - return new AddColumnOp(columnDefinition, columnPosition, rollupName, properties); + return new AddColumnOp(columnDefinitionWithPath.columnDefinition, columnDefinitionWithPath.columnPath, + columnPosition, rollupName, properties); } @Override @@ -6133,17 +6209,17 @@ public AlterTableOp visitAddColumnsClause(AddColumnsClauseContext ctx) { @Override public AlterTableOp visitDropColumnClause(DropColumnClauseContext ctx) { - String columnName = ctx.name.getText(); + ColumnPath columnPath = parseColumnPath(ctx.name, visitQualifiedName(ctx.name)); String rollupName = ctx.fromRollup() != null ? ctx.fromRollup().rollup.getText() : null; Map properties = ctx.properties != null ? Maps.newHashMap(visitPropertyClause(ctx.properties)) : Maps.newHashMap(); - return new DropColumnOp(columnName, rollupName, properties); + return new DropColumnOp(columnPath, rollupName, properties); } @Override public AlterTableOp visitModifyColumnClause(ModifyColumnClauseContext ctx) { - ColumnDefinition columnDefinition = visitColumnDef(ctx.columnDef()); + ColumnDefinitionWithPath columnDefinitionWithPath = visitColumnDefWithPath(ctx.columnDefWithPath()); ColumnPosition columnPosition = null; if (ctx.columnPosition() != null) { if (ctx.columnPosition().FIRST() != null) { @@ -6156,12 +6232,14 @@ public AlterTableOp visitModifyColumnClause(ModifyColumnClauseContext ctx) { Map properties = ctx.properties != null ? Maps.newHashMap(visitPropertyClause(ctx.properties)) : Maps.newHashMap(); - return new ModifyColumnOp(columnDefinition, columnPosition, rollupName, properties); + return new ModifyColumnOp(columnDefinitionWithPath.columnDefinition, columnDefinitionWithPath.columnPath, + columnPosition, rollupName, properties); } @Override public AlterTableOp visitReorderColumnsClause(ReorderColumnsClauseContext ctx) { List columnsByPos = visitIdentifierList(ctx.identifierList()); + columnsByPos.forEach(column -> requireNonEmptyColumnIdentifier(ctx.identifierList(), column)); String rollupName = ctx.fromRollup() != null ? ctx.fromRollup().rollup.getText() : null; Map properties = ctx.properties != null ? Maps.newHashMap(visitPropertyClause(ctx.properties)) @@ -6359,7 +6437,7 @@ public AlterTableOp visitRenamePartitionClause(RenamePartitionClauseContext ctx) @Override public AlterTableOp visitRenameColumnClause(RenameColumnClauseContext ctx) { - return new RenameColumnOp(ctx.name.getText(), ctx.newName.getText()); + return new RenameColumnOp(parseColumnPath(ctx.name, visitQualifiedName(ctx.name)), ctx.newName.getText()); } @Override @@ -6467,9 +6545,9 @@ public AlterTableOp visitModifyTableCommentClause(ModifyTableCommentClauseContex @Override public AlterTableOp visitModifyColumnCommentClause(ModifyColumnCommentClauseContext ctx) { - String columnName = ctx.name.getText(); - String comment = stripQuotes(ctx.STRING_LITERAL().getText()); - return new ModifyColumnCommentOp(columnName, comment); + ColumnPath columnPath = parseColumnPath(ctx.name, visitQualifiedName(ctx.name)); + String comment = SqlLiteralUtils.parseStringLiteral(ctx.STRING_LITERAL().getText()); + return new ModifyColumnCommentOp(columnPath, comment); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java index 7806201a75dbbd..63a8e9557b1144 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderAssistant.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.util.SqlLiteralUtils; import com.google.common.collect.ImmutableSet; @@ -42,45 +43,7 @@ private LogicalPlanBuilderAssistant() { * EscapeBackSlash such \n, \t */ public static String escapeBackSlash(String str) { - StringBuilder sb = new StringBuilder(); - int strLen = str.length(); - for (int i = 0; i < strLen; ++i) { - char c = str.charAt(i); - if (c == '\\' && (i + 1) < strLen) { - switch (str.charAt(i + 1)) { - case 'n': - sb.append('\n'); - break; - case 't': - sb.append('\t'); - break; - case 'r': - sb.append('\r'); - break; - case 'b': - sb.append('\b'); - break; - case '0': - sb.append('\0'); // Ascii null - break; - case 'Z': // ^Z must be escaped on Win32 - sb.append('\032'); - break; - case '_': - case '%': - sb.append('\\'); // remember prefix for wildcard - sb.append(str.charAt(i + 1)); - break; - default: - sb.append(str.charAt(i + 1)); - break; - } - i++; - } else { - sb.append(c); - } - } - return sb.toString(); + return SqlLiteralUtils.unescapeBackSlash(str); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 83f978fff948b9..c7a7bed074e9f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -59,6 +59,7 @@ import java.util.BitSet; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -300,13 +301,14 @@ public List> parseMultiple(String sql, } public Expression parseExpression(String expression) { - if (isSimpleIdentifier(expression)) { + if (isValidUnquotedIdentifier(expression)) { return new UnboundSlot(expression); } return parse(expression, DorisParser::expressionWithEof); } - private static boolean isSimpleIdentifier(String expression) { + /** Return whether the text can be emitted as an unquoted Nereids identifier. */ + public static boolean isValidUnquotedIdentifier(String expression) { if (expression == null || expression.isEmpty()) { return false; } @@ -323,7 +325,7 @@ private static boolean isSimpleIdentifier(String expression) { if (!hasLetter) { return false; } - String upperCase = expression.toUpperCase(); + String upperCase = expression.toUpperCase(Locale.ROOT); return (NON_RESERVED_KEYWORDS.contains(upperCase) || !LITERAL_TOKENS.containsKey(upperCase)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 616bd387616916..823231aa317482 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -71,9 +71,11 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -447,53 +449,78 @@ public PhysicalProperties visitPhysicalSetOperation(PhysicalSetOperation setOper return PhysicalProperties.GATHER; } - // TODO: open comment when support `enable_local_shuffle_planner` - // int distributeToChildIndex - // = setOperation.getMutableState(PhysicalSetOperation.DISTRIBUTE_TO_CHILD_INDEX).orElse(-1); - // if (distributeToChildIndex >= 0 - // && childrenDistribution.get(distributeToChildIndex) instanceof DistributionSpecHash) { - // DistributionSpecHash childDistribution - // = (DistributionSpecHash) childrenDistribution.get(distributeToChildIndex); - // List childToIndex = setOperation.getRegularChildrenOutputs().get(distributeToChildIndex); - // Map idToOutputIndex = new LinkedHashMap<>(); - // for (int j = 0; j < childToIndex.size(); j++) { - // idToOutputIndex.put(childToIndex.get(j).getExprId(), j); - // } - // - // List orderedShuffledColumns = childDistribution.getOrderedShuffledColumns(); - // List setOperationDistributeColumnIds = new ArrayList<>(); - // for (ExprId tableDistributeColumnId : orderedShuffledColumns) { - // Integer index = idToOutputIndex.get(tableDistributeColumnId); - // if (index == null) { - // break; - // } - // setOperationDistributeColumnIds.add(setOperation.getOutput().get(index).getExprId()); - // } - // // check whether the set operation output all distribution columns of the child - // if (setOperationDistributeColumnIds.size() == orderedShuffledColumns.size()) { - // boolean isUnion = setOperation instanceof Union; - // boolean shuffleToRight = distributeToChildIndex > 0; - // if (!isUnion && shuffleToRight) { - // return new PhysicalProperties( - // new DistributionSpecHash( - // setOperationDistributeColumnIds, - // ShuffleType.EXECUTION_BUCKETED - // ) - // ); - // } else { - // // keep the distribution as the child - // return new PhysicalProperties( - // new DistributionSpecHash( - // setOperationDistributeColumnIds, - // childDistribution.getShuffleType(), - // childDistribution.getTableId(), - // childDistribution.getSelectedIndexId(), - // childDistribution.getPartitionIds() - // ) - // ); - // } - // } - // } + // After ChildrenPropertiesRegulator the children distributions are already legal, so the + // output is derived by describing what the children provide: + // 1. one or more NATURAL children: output the first NATURAL child's distribution + // (several NATURAL children behave like colocate); + // 2. no NATURAL but some STORAGE_BUCKETED child: output its STORAGE_BUCKETED distribution; + // 3. all EXECUTION_BUCKETED children: output the execution hash (the generic loop below); + // 4. anything else (e.g. random): output a non-specific property (also below). + // When the basic child does not directly output its shuffle columns, or the children do + // not agree, this falls through to the generic loop below, which is equivalence-set aware + // and checks that every child maps its shuffle columns to the same set-operation output + // positions before it claims a bucketed output. The basic child is recomputed from the + // children distributions instead of being carried as mutable planner state, because mutable + // state does not survive the with-copies in chooseBestPlan() and the + // RecomputePhysicalPropertiesPostProcessor re-derivation, while this recomputation is + // deterministic on any copy of the plan. + int distributeToChildIndex = -1; + int firstStorageBucketedIndex = -1; + for (int i = 0; i < childrenDistribution.size(); i++) { + if (childrenDistribution.get(i) instanceof DistributionSpecHash) { + ShuffleType childShuffleType + = ((DistributionSpecHash) childrenDistribution.get(i)).getShuffleType(); + if (childShuffleType == ShuffleType.NATURAL) { + distributeToChildIndex = i; + break; + } else if (childShuffleType == ShuffleType.STORAGE_BUCKETED + && firstStorageBucketedIndex < 0) { + firstStorageBucketedIndex = i; + } + } + } + if (distributeToChildIndex < 0) { + distributeToChildIndex = firstStorageBucketedIndex; + } + if (distributeToChildIndex >= 0) { + DistributionSpecHash childDistribution + = (DistributionSpecHash) childrenDistribution.get(distributeToChildIndex); + List childToIndex = setOperation.getRegularChildrenOutputs().get(distributeToChildIndex); + Map idToOutputIndex = new LinkedHashMap<>(); + for (int j = 0; j < childToIndex.size(); j++) { + idToOutputIndex.put(childToIndex.get(j).getExprId(), j); + } + + List orderedShuffledColumns = childDistribution.getOrderedShuffledColumns(); + List setOperationDistributeColumnIds = new ArrayList<>(); + for (ExprId tableDistributeColumnId : orderedShuffledColumns) { + Integer index = idToOutputIndex.get(tableDistributeColumnId); + if (index == null) { + break; + } + setOperationDistributeColumnIds.add(setOperation.getOutput().get(index).getExprId()); + } + // check whether the set operation output all distribution columns of the child + if (setOperationDistributeColumnIds.size() == orderedShuffledColumns.size()) { + // Keep the basic child's specific storage layout as the set operation output. When + // the basic child is on the right (shuffleToRight) the output rows are physically + // placed by the right child's storage bucket function, so advertising that layout is + // truthful: a parent can co-locate this set operation against the same layout, while + // a sibling on a different layout is re-aligned rather than wrongly co-located. (The + // earlier "Can not find tablet ... in the bucket" failure came from advertising a + // layout-less EXECUTION_BUCKETED here, which erased the table id so two different- + // layout set operations looked co-locatable; preserving the layout fixes that.) + return new PhysicalProperties( + new DistributionSpecHash( + setOperationDistributeColumnIds, + childDistribution.getShuffleType(), + childDistribution.getTableId(), + childDistribution.getSelectedIndexId(), + childDistribution.getPartitionIds() + ) + ); + } + } for (int i = 0; i < childrenDistribution.size(); i++) { DistributionSpec childDistribution = childrenDistribution.get(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java index e5e0d9d1bd0d28..6aa9d01c7cc709 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java @@ -57,6 +57,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -651,83 +652,103 @@ public List> visitPhysicalSetOperation(PhysicalSetOpera } else if (requiredDistributionSpec instanceof DistributionSpecHash) { // TODO: should use the most common hash spec as basic DistributionSpecHash basic = (DistributionSpecHash) requiredDistributionSpec; - // TODO: open comment when support `enable_local_shuffle_planner` - // int bucketShuffleBasicIndex = -1; - // double basicRowCount = -1; - - // find the bucket shuffle basic index - // try { - // ImmutableSet supportBucketShuffleTypes = ImmutableSet.of( - // ShuffleType.NATURAL, - // ShuffleType.STORAGE_BUCKETED - // ); - // for (int i = 0; i < originChildrenProperties.size(); i++) { - // PhysicalProperties originChildrenProperty = originChildrenProperties.get(i); - // DistributionSpec childDistribution = originChildrenProperty.getDistributionSpec(); - // if (childDistribution instanceof DistributionSpecHash - // && supportBucketShuffleTypes.contains( - // ((DistributionSpecHash) childDistribution).getShuffleType()) - // && !(isBucketShuffleDownGrade(setOperation.child(i)))) { - // Statistics stats = setOperation.child(i).getStats(); - // double rowCount = stats.getRowCount(); - // if (rowCount > basicRowCount) { - // basicRowCount = rowCount; - // bucketShuffleBasicIndex = i; - // } - // } - // } - // } catch (Throwable t) { - // // catch stats exception - // LOG.warn("Can not find the most (bucket num, rowCount): " + t, t); - // bucketShuffleBasicIndex = -1; - // } - - // use bucket shuffle - // if (bucketShuffleBasicIndex >= 0) { - // DistributionSpecHash notShuffleSideRequire - // = (DistributionSpecHash) requiredProperties.get(bucketShuffleBasicIndex) - // .getDistributionSpec(); - // - // DistributionSpecHash notNeedShuffleOutput - // = (DistributionSpecHash) originChildrenProperties.get(bucketShuffleBasicIndex) - // .getDistributionSpec(); - // - // for (int i = 0; i < originChildrenProperties.size(); i++) { - // DistributionSpecHash current - // = (DistributionSpecHash) originChildrenProperties.get(i).getDistributionSpec(); - // if (i == bucketShuffleBasicIndex) { - // continue; - // } - // - // DistributionSpecHash currentRequire - // = (DistributionSpecHash) requiredProperties.get(i).getDistributionSpec(); - // - // PhysicalProperties target = calAnotherSideRequired( - // ShuffleType.STORAGE_BUCKETED, - // notNeedShuffleOutput, current, - // notShuffleSideRequire, - // currentRequire); - // updateChildEnforceAndCost(i, target); - // } - // setOperation.setMutableState( - // PhysicalSetOperation.DISTRIBUTE_TO_CHILD_INDEX, bucketShuffleBasicIndex); - // use partitioned shuffle - // } else { - for (int i = 0; i < originChildrenProperties.size(); i++) { - DistributionSpecHash current - = (DistributionSpecHash) originChildrenProperties.get(i).getDistributionSpec(); - if (current.getShuffleType() != ShuffleType.EXECUTION_BUCKETED - || !bothSideShuffleKeysAreSameOrder(basic, current, - (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), - (DistributionSpecHash) requiredProperties.get(i).getDistributionSpec())) { - PhysicalProperties target = calAnotherSideRequired( - ShuffleType.EXECUTION_BUCKETED, basic, current, - (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), - (DistributionSpecHash) requiredProperties.get(i).getDistributionSpec()); + int bucketShuffleBasicIndex = -1; + double basicRowCount = -1; + + // find the bucket shuffle basic index: the largest natural / storage-bucketed child + // keeps its bucket distribution, every other child is bucket-shuffled to it. + // RequestPropertyDeriver only asks ShuffleType.REQUIRE when set-op bucket shuffle + // is allowed, so the required shuffle type is the single source of truth here: + // for any other required type keep bucketShuffleBasicIndex = -1 and fall back to + // the execution-bucketed (partitioned) shuffle below. + // isBucketShuffleDownGrade reuses the join-side heuristics on purpose, including + // the enable_bucket_shuffle_join switch and bucket_shuffle_downgrade_ratio: bucket + // shuffle for set operation belongs to the same optimization family as bucket + // shuffle join, so the join switches govern both instead of introducing a separate + // session variable. + if (basic.getShuffleType() == ShuffleType.REQUIRE) { + try { + ImmutableSet supportBucketShuffleTypes = ImmutableSet.of( + ShuffleType.NATURAL, + ShuffleType.STORAGE_BUCKETED + ); + for (int i = 0; i < originChildrenProperties.size(); i++) { + PhysicalProperties originChildrenProperty = originChildrenProperties.get(i); + DistributionSpec childDistribution = originChildrenProperty.getDistributionSpec(); + // The table id is deliberately not checked here: DistributionSpecHash.satisfy + // aligns the other children by shuffle type and columns regardless of the table + // id, and a basic child with an unknown layout (a hash join output with the table + // id cleared to -1 by withShuffleTypeAndForbidColocateJoin) produces a + // STORAGE_BUCKETED output, which couldColocateJoin never co-locates (it requires + // NATURAL on both sides) so it cannot mislead a parent into a wrong co-location. + if (childDistribution instanceof DistributionSpecHash + && supportBucketShuffleTypes.contains( + ((DistributionSpecHash) childDistribution).getShuffleType()) + && canMapBucketKeysToRequire((DistributionSpecHash) childDistribution, + (DistributionSpecHash) requiredProperties.get(i).getDistributionSpec()) + && !(isBucketShuffleDownGrade(setOperation.child(i)))) { + Statistics stats = setOperation.child(i).getStats(); + double rowCount = stats.getRowCount(); + if (rowCount > basicRowCount) { + basicRowCount = rowCount; + bucketShuffleBasicIndex = i; + } + } + } + } catch (Throwable t) { + // catch stats exception + LOG.warn("Can not find the most (bucket num, rowCount): " + t, t); + bucketShuffleBasicIndex = -1; + } + } + + if (bucketShuffleBasicIndex >= 0) { + // use bucket shuffle + DistributionSpecHash notShuffleSideRequire + = (DistributionSpecHash) requiredProperties.get(bucketShuffleBasicIndex) + .getDistributionSpec(); + + DistributionSpecHash notNeedShuffleOutput + = (DistributionSpecHash) originChildrenProperties.get(bucketShuffleBasicIndex) + .getDistributionSpec(); + + for (int i = 0; i < originChildrenProperties.size(); i++) { + if (i == bucketShuffleBasicIndex) { + continue; + } + + DistributionSpecHash currentRequire + = (DistributionSpecHash) requiredProperties.get(i).getDistributionSpec(); + + // The enforced child is bucket-shuffled to the basic child's buckets by the + // storage hash on shuffleSideIds. Only the shuffle type and the column order + // carry the alignment: DistributionSpecHash.satisfy compares shuffle type and + // columns (the equivalence set), never the storage layout, and the set operation + // output is STORAGE_BUCKETED which couldColocateJoin never co-locates, so the + // basic child's table / index / partition ids are inert here. + List shuffleSideIds = calAnotherSideRequiredShuffleIds( + notNeedShuffleOutput, notShuffleSideRequire, currentRequire); + PhysicalProperties target = new PhysicalProperties( + new DistributionSpecHash(shuffleSideIds, ShuffleType.STORAGE_BUCKETED)); updateChildEnforceAndCost(i, target); } + } else { + // use partitioned shuffle + for (int i = 0; i < originChildrenProperties.size(); i++) { + DistributionSpecHash current + = (DistributionSpecHash) originChildrenProperties.get(i).getDistributionSpec(); + if (current.getShuffleType() != ShuffleType.EXECUTION_BUCKETED + || !bothSideShuffleKeysAreSameOrder(basic, current, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(i).getDistributionSpec())) { + PhysicalProperties target = calAnotherSideRequired( + ShuffleType.EXECUTION_BUCKETED, basic, current, + (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(), + (DistributionSpecHash) requiredProperties.get(i).getDistributionSpec()); + updateChildEnforceAndCost(i, target); + } + } } - // } } return ImmutableList.of(originChildrenProperties); } @@ -801,6 +822,32 @@ private boolean bothSideShuffleKeysAreSameOrder( } } + /** + * Whether every bucket key of the candidate basic child can be mapped into the child's + * required hash columns (directly or through its equivalence sets). When the candidate's + * bucket key is wider than the set operation output (e.g. a table bucketed by (k, v) + * feeding INTERSECT on k only), the mapping is impossible and choosing it as the basic + * child would fail calAnotherSideRequiredShuffleIds, so the caller falls back to the + * execution-bucketed shuffle instead. + */ + private boolean canMapBucketKeysToRequire(DistributionSpecHash childOutput, DistributionSpecHash childRequired) { + for (ExprId scanId : childOutput.getOrderedShuffledColumns()) { + int index = childRequired.getOrderedShuffledColumns().indexOf(scanId); + if (index == -1) { + for (ExprId alternativeExpr : childOutput.getEquivalenceExprIdsOf(scanId)) { + index = childRequired.getOrderedShuffledColumns().indexOf(alternativeExpr); + if (index != -1) { + break; + } + } + } + if (index == -1) { + return false; + } + } + return true; + } + /** * calculate the shuffle side hash key right orders. * For example, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java index 1e0654aec46e1d..6d21c1231f1d65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderKey.java @@ -112,4 +112,8 @@ public boolean equals(Object o) { OrderKey that = (OrderKey) o; return isAsc == that.isAsc() && nullFirst == that.isNullFirst() && expr.equals(that.getExpr()); } + + public String shapeInfo() { + return expr.shapeInfo() + (isAsc ? " asc" : " desc") + (nullFirst ? " null first" : ""); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java index 70f2b51665b740..6ea2601bbee73c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java @@ -67,6 +67,7 @@ import org.apache.doris.nereids.util.AggregateUtils; import org.apache.doris.nereids.util.JoinUtils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.Statistics; import com.google.common.base.Preconditions; @@ -333,20 +334,18 @@ public Void visitPhysicalSetOperation(PhysicalSetOperation setOperation, PlanCon if (distributionRequestFromParent instanceof DistributionSpecHash) { // shuffle according to parent require DistributionSpecHash distributionSpecHash = (DistributionSpecHash) distributionRequestFromParent; - addRequestPropertyToChildren(createHashRequestAccordingToParent( - setOperation, distributionSpecHash, context)); + addRequestPropertyToChildren(downgradeRequireWhenBucketShuffleNotAllowed( + createHashRequestAccordingToParent(setOperation, distributionSpecHash, context))); } else { // shuffle all column // TODO: for wide table, may be we should add a upper limit of shuffle columns - - // TODO: open comment when support `enable_local_shuffle_planner` and change to REQUIRE - // intersect/except always need hash distribution, we use REQUIRE to auto select - // bucket shuffle or execution shuffle + ShuffleType setOperationShuffleType = setOperationBucketShuffleAllowed() + ? ShuffleType.REQUIRE : ShuffleType.EXECUTION_BUCKETED; addRequestPropertyToChildren(setOperation.getRegularChildrenOutputs().stream() .map(childOutputs -> childOutputs.stream() .map(SlotReference::getExprId) .collect(ImmutableList.toImmutableList())) - .map(l -> PhysicalProperties.createHash(l, ShuffleType.EXECUTION_BUCKETED)) + .map(l -> PhysicalProperties.createHash(l, setOperationShuffleType)) .collect(Collectors.toList())); } return null; @@ -366,9 +365,8 @@ public Void visitPhysicalUnion(PhysicalUnion union, PlanContext context) { DistributionSpec distributionRequestFromParent = requestPropertyFromParent.getDistributionSpec(); if (distributionRequestFromParent instanceof DistributionSpecHash) { DistributionSpecHash distributionSpecHash = (DistributionSpecHash) distributionRequestFromParent; - List requestHash - = createHashRequestAccordingToParent(union, distributionSpecHash, context); - addRequestPropertyToChildren(requestHash); + addRequestPropertyToChildren(downgradeRequireWhenBucketShuffleNotAllowed( + createHashRequestAccordingToParent(union, distributionSpecHash, context))); } } @@ -581,6 +579,43 @@ private boolean shouldUseParent(List parentHashExprIds, PhysicalHashAggr return combinedNdv > AggregateUtils.LOW_NDV_THRESHOLD; } + /** + * A ShuffleType.REQUIRE request lets ChildrenPropertiesRegulator choose the bucket + * shuffle alternative for the set operation. That needs either no local shuffle at all + * (every pipeline runs a single task per instance, so the bucket alignment holds + * naturally) or the FE local-shuffle planner (which plans the correct bucket-hash + * local exchanges): with the BE-planned local shuffle the backend cannot infer the + * correct local shuffle type for the set sink/probe and computes wrong results. + * It also requires the nereids distribute planner: the legacy coordinator only + * supports bucket-shuffle-partitioned sinks whose dest fragment contains a bucket + * shuffle join. + */ + private boolean setOperationBucketShuffleAllowed() { + return connectContext != null + && SessionVariable.canUseNereidsDistributePlanner(connectContext) + && (!connectContext.getSessionVariable().isEnableLocalShuffle() + || connectContext.getSessionVariable().isEnableLocalShufflePlanner()); + } + + /** + * The parent may pass ShuffleType.REQUIRE down through + * {@link #createHashRequestAccordingToParent}; downgrade it to EXECUTION_BUCKETED so the + * regulator does not pick the bucket shuffle alternative when it is not allowed. + */ + private List downgradeRequireWhenBucketShuffleNotAllowed( + List requests) { + if (setOperationBucketShuffleAllowed()) { + return requests; + } + return requests.stream().map(request -> { + DistributionSpecHash requestHash = (DistributionSpecHash) request.getDistributionSpec(); + return requestHash.getShuffleType() == ShuffleType.REQUIRE + ? PhysicalProperties.createHash( + requestHash.getOrderedShuffledColumns(), ShuffleType.EXECUTION_BUCKETED) + : request; + }).collect(Collectors.toList()); + } + private List createHashRequestAccordingToParent( SetOperation setOperation, DistributionSpecHash distributionRequestFromParent, PlanContext context) { List requiredPropertyList = diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java index 7b5e88060deded..93a31c61f1a91b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java @@ -40,6 +40,7 @@ import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; @@ -163,7 +164,8 @@ private LogicalPlan normalizeAgg(LogicalAggregate aggregate, Optional groupingByExprs = Utils.fastToImmutableSet(aggregate.getGroupByExpressions()); // collect all trivial-agg - List aggregateOutput = aggregate.getOutputExpressions(); + List aggregateOutput = normalizeMultiColumnDistinctCount( + aggregate.getOutputExpressions()); Map> aggFuncs = CollectNonWindowedAggFuncsWithSessionVar.collect(aggregateOutput); @@ -173,6 +175,12 @@ private LogicalPlan normalizeAgg(LogicalAggregate aggregate, Optional needPushDownSelfExprs = ImmutableSet.builder(); ImmutableSet.Builder needPushDownInputs = ImmutableSet.builder(); for (AggregateFunction aggFunc : aggFuncs.keySet()) { + for (Expression child : aggFunc.children()) { + if (ExpressionUtils.hasNonWindowAggregateFunction(child)) { + throw new AnalysisException( + "aggregate function cannot contain aggregate parameters"); + } + } if (!aggFunc.isDistinct()) { for (Expression arg : aggFunc.children()) { // should not push down literal under aggregate @@ -257,11 +265,6 @@ private LogicalPlan normalizeAgg(LogicalAggregate aggregate, Optional normalizedAggFuncs = bottomSlotContext.normalizeToUseSlotRef(SessionVarGuardExpr.getExprWithGuard(aggFuncs)); - if (normalizedAggFuncs.stream().anyMatch(agg -> !agg.children().isEmpty() - && agg.child(0).containsType(AggregateFunction.class))) { - throw new AnalysisException( - "aggregate function cannot contain aggregate parameters"); - } // build normalized agg output NormalizeToSlotContext normalizedAggFuncsToSlotContext = @@ -447,6 +450,28 @@ private LogicalPlan normalizeAgg(LogicalAggregate aggregate, Optional(bottomProjectsBuilder.build(), newAggregate))); } + private List normalizeMultiColumnDistinctCount(List aggregateOutput) { + // Multi-column distinct counts treat arguments as a set. Remove duplicates and canonicalize equivalent + // counts to the first argument order so the structural equality used below can share one aggregate result. + Map, Count> distinctArgumentsToCount = new HashMap<>(); + return ExpressionUtils.rewriteDownShortCircuit(aggregateOutput, expression -> { + if (!(expression instanceof Count)) { + return expression; + } + Count count = (Count) expression; + if (!count.isDistinct() || count.arity() <= 1) { + return count; + } + ImmutableSet distinctArguments = ImmutableSet.copyOf(count.getDistinctArguments()); + Count normalizedCount = distinctArgumentsToCount.get(distinctArguments); + if (normalizedCount == null) { + normalizedCount = count.withDistinctAndChildren(true, ImmutableList.copyOf(distinctArguments)); + distinctArgumentsToCount.put(distinctArguments, normalizedCount); + } + return count.withDistinctAndChildren(true, normalizedCount.children()); + }); + } + private List normalizeOutput(List aggregateOutput, NormalizeToSlotContext groupByToSlotContext, NormalizeToSlotContext argsOfAggFuncNeedPushDownContext, NormalizeToSlotContext normalizedAggFuncsToSlotContext) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index 975cfaf973c408..6099699bd467c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.Or; @@ -570,6 +571,8 @@ private LogicalAggregate storageLayerAggregate( Map, PushDownAggOp> supportedAgg = PushDownAggOp.supportedFunctions(); boolean containsCount = false; + boolean containsCountStar = false; + boolean countHasCastArgument = false; Set checkNullSlots = new HashSet<>(); Set expressionAfterProject = new HashSet<>(); @@ -585,12 +588,15 @@ private LogicalAggregate storageLayerAggregate( // Check if contains Count function if (functionClass.equals(Count.class)) { containsCount = true; - if (!function.getArguments().isEmpty()) { + if (function.getArguments().isEmpty()) { + containsCountStar = true; + } else { Expression arg0 = function.getArguments().get(0); if (arg0 instanceof SlotReference) { checkNullSlots.add((SlotReference) arg0); expressionAfterProject.add(arg0); } else if (arg0 instanceof Cast) { + countHasCastArgument = true; Expression child0 = arg0.child(0); if (child0 instanceof SlotReference) { checkNullSlots.add((SlotReference) child0); @@ -667,6 +673,7 @@ private LogicalAggregate storageLayerAggregate( return canNotPush; } else { if (needCheckSlotNull) { + countHasCastArgument = true; checkNullSlots.add((SlotReference) argument.child(0)); } } @@ -677,6 +684,16 @@ private LogicalAggregate storageLayerAggregate( argumentsOfAggregateFunction = processedExpressions; } + // File aggregate metadata can describe COUNT(*) or COUNT(file_column), but it cannot + // describe the CAST wrapped around a COUNT argument. Dropping that CAST is incorrect even + // when the source column is NOT NULL. For example, a non-null DOUBLE value outside the INT + // range becomes NULL for CAST(double_col AS INT), so COUNT(CAST(double_col AS INT)) must + // exclude it while a footer-level COUNT(double_col) would include it. Keep OLAP's existing + // storage-layer behavior unchanged, and make external files evaluate the CAST normally. + if (logicalScan instanceof LogicalFileScan && countHasCastArgument) { + return canNotPush; + } + Set pushDownAggOps = functionClasses.stream() .map(supportedAgg::get) .collect(Collectors.toSet()); @@ -690,6 +707,12 @@ private LogicalAggregate storageLayerAggregate( List usedSlotInTable = (List) Project.findProject(aggUsedSlots, logicalScan.getOutput()); + // COUNT(*) has no aggregate arguments, even though later column pruning retains one + // arbitrary scan slot. Preserve the semantic arguments here so the BE never needs to infer + // COUNT(col) from the post-pruning scan shape. + List countArgumentExprIds = mergeOp == PushDownAggOp.COUNT + ? usedSlotInTable.stream().map(SlotReference::getExprId).collect(Collectors.toList()) + : ImmutableList.of(); for (SlotReference slot : usedSlotInTable) { Optional optionalColumn = slot.getOriginalColumn(); @@ -720,10 +743,21 @@ private LogicalAggregate storageLayerAggregate( } } if (mergeOp == PushDownAggOp.COUNT || mergeOp == PushDownAggOp.MIX) { - // NULL value behavior in `count` function is zero, so - // we should not use row_count to speed up query. the col - // must be not null - if (column.isAllowNull() && checkNullSlots.contains(slot)) { + if (logicalScan instanceof LogicalFileScan && mergeOp == PushDownAggOp.COUNT + && containsCountStar && column.isAllowNull() && checkNullSlots.contains(slot)) { + // One metadata cardinality cannot represent both COUNT(*) and the smaller + // COUNT(nullable_col); synthetic rows would make one upper aggregate wrong. + return canNotPush; + } + // Nullable file COUNT is exact only when this query is routed to FileScannerV2, + // which carries the semantic argument and counts definition levels. Gating on the + // session switch keeps V1 on its original full-column evaluation path. + boolean supportsNullableFileCount = logicalScan instanceof LogicalFileScan + && mergeOp == PushDownAggOp.COUNT + && cascadesContext.getConnectContext() != null + && cascadesContext.getConnectContext().getSessionVariable().enableFileScannerV2; + if (column.isAllowNull() && checkNullSlots.contains(slot) + && !supportsNullableFileCount) { return canNotPush; } } @@ -738,11 +772,12 @@ private LogicalAggregate storageLayerAggregate( if (project != null) { return aggregate.withChildren(ImmutableList.of( project.withChildren( - ImmutableList.of(new PhysicalStorageLayerAggregate(physicalScan, mergeOp))) + ImmutableList.of(new PhysicalStorageLayerAggregate( + physicalScan, mergeOp, countArgumentExprIds))) )); } else { return aggregate.withChildren(ImmutableList.of( - new PhysicalStorageLayerAggregate(physicalScan, mergeOp) + new PhysicalStorageLayerAggregate(physicalScan, mergeOp, countArgumentExprIds) )); } @@ -754,11 +789,12 @@ private LogicalAggregate storageLayerAggregate( if (project != null) { return aggregate.withChildren(ImmutableList.of( project.withChildren( - ImmutableList.of(new PhysicalStorageLayerAggregate(physicalScan, mergeOp))) + ImmutableList.of(new PhysicalStorageLayerAggregate( + physicalScan, mergeOp, countArgumentExprIds))) )); } else { return aggregate.withChildren(ImmutableList.of( - new PhysicalStorageLayerAggregate(physicalScan, mergeOp) + new PhysicalStorageLayerAggregate(physicalScan, mergeOp, countArgumentExprIds) )); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java index f3822215e8c4c7..83db60b098a238 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import org.apache.commons.collections4.CollectionUtils; @@ -94,8 +95,9 @@ private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable, Optional> sortedPartitionRanges = Optional.empty(); boolean enableBinarySearch = ctx.getConnectContext() == null || ctx.getConnectContext().getSessionVariable().enableBinarySearchFilteringPartitions; - if (enableBinarySearch) { - sortedPartitionRanges = (Optional) externalTable.getSortedPartitionRanges(scan); + if (enableBinarySearch && !nameToPartitionItem.isEmpty()) { + sortedPartitionRanges = scan.getSelectedPartitions().sortedPartitionRanges + .or(() -> Optional.ofNullable(SortedPartitionRanges.build(nameToPartitionItem))); } PartitionPruneResult result = PartitionPruner.pruneWithResult( partitionSlots, filter.getPredicate(), nameToPartitionItem, ctx, @@ -103,7 +105,13 @@ private SelectedPartitions pruneExternalPartitions(ExternalTable externalTable, List prunedPartitions = new ArrayList<>(result.partitions); for (String name : prunedPartitions) { - selectedPartitionItems.put(name, nameToPartitionItem.get(name)); + PartitionItem item = nameToPartitionItem.get(name); + // Both nameToPartitionItem and sortedPartitionRanges now come from the same frozen + // snapshot, so a missing item is an invariant violation rather than a partition to + // skip. Failing here surfaces the bug instead of silently returning a partial scan. + Preconditions.checkState(item != null, + "pruned partition %s is missing in the selected partitions snapshot", name); + selectedPartitionItems.put(name, item); } return new SelectedPartitions(nameToPartitionItem.size(), selectedPartitionItems, true, result.hasPartitionPredicate); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java index 3f9ad609744e21..dd09b2b52a3dae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java @@ -64,7 +64,7 @@ public List buildRules() { } else { return funcs.stream() .allMatch(f -> (f instanceof Min || f instanceof Max || f instanceof Sum - || f instanceof Count) && f.isDistinct() + || f instanceof Count) && f.isDistinct() && f.arity() == 1 && f.child(0) instanceof Slot); } }) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java index 748a17f99b3d11..088de2e0f0668f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java @@ -776,6 +776,43 @@ private Plan genAggregate(Plan child, PushDownAggContext context) { if (isPushDisabledByVariable(context)) { return child; } + // Forbid creating a scalar aggregate (aggregate without GROUP BY keys) during + // push-down. A scalar aggregate emits 1 row even when its input is empty + // (SQL standard: SELECT COUNT(*) FROM empty_table returns 1 row with 0). + // When placed below a join, that 1 row joins with rows from the other side, + // producing phantom rows that do not exist in the original query. + // + // This guard is placed at the aggregate-creation boundary rather than upstream + // (e.g. visitLogicalJoin or createContextFromProject) because context.groupKeys + // can change during intermediate rewrites. Checking groupKeys.isEmpty() early + // would either fire on a stale empty state (missing a later filter that adds + // keys) or fail to fire because a constant key has not yet been resolved to + // empty input slots. Example plan that reaches genAggregate with groupKeys=[]: + // + // Aggregate(group=[k], sum(z)) + // CrossJoin + // Project(1 AS k, A.v + B.v AS z) + // InnerJoin(A.id = B.id) + // Scan A + // Scan B + // Scan R + // + // Walk: + // 1. visitLogicalJoin(crossJoin): parentContext.groupKeys=[k] + // → fillGroupByKeys puts k in leftChildGroupByKeys → forOneBranch succeeds + // 2. visitLogicalProject: createContextFromProject maps k through "1 AS k" + // → literal 1 has no input slots → newContext.groupKeys=[] + // 3. visitLogicalJoin(innerJoin): sum(A.v+B.v) spans both sides + // → toLeft=false, toRight=false → falls back to genAggregate with groupKeys=[] + // → without this guard, creates scalar agg below crossJoin → phantom rows + // + // If the guard were at visitLogicalJoin step 1, groupKeys=[k] (non-empty) would + // pass. If at createContextFromProject step 2, it would block before a downstream + // filter had the chance to add keys. Only genAggregate sees the final state. + // See regression test: scalar_agg_pushdown.groovy + if (context.getGroupKeys().isEmpty()) { + return child; + } if (checkStats(child, context) || isPushEnabledByVariable(context)) { List aggOutputExpressions = new ArrayList<>(); for (AggregateFunction func : context.getAggFunctions()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AggregateExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AggregateExpression.java index 05ad20655ddec9..aa5973ceac552d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AggregateExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AggregateExpression.java @@ -108,6 +108,15 @@ public String computeToSql() { } } + @Override + public String shapeInfo() { + if (aggregateParam.aggMode.productAggregateBuffer) { + return "partial_" + function.shapeInfo(); + } else { + return function.shapeInfo(); + } + } + @Override public String toString() { AggMode aggMode = aggregateParam.aggMode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index edb416014a6fc4..7a513d213cb601 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -129,6 +129,11 @@ public String computeToSql() { return child().toSql() + " AS `" + name.get() + "`"; } + @Override + public String shapeInfo() { + return child().shapeInfo() + " AS `" + name.get() + "`"; + } + @Override public boolean nullable() throws UnboundException { return child().nullable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java index bfbb184ec30191..c4b836b31e05c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java @@ -96,6 +96,11 @@ public String computeToSql() { return child(0).toSql(); } + @Override + public String shapeInfo() { + return child(0).shapeInfo(); + } + @Override public Slot toSlot() { return new ArrayItemSlot(exprId, name, getDataType(), nullable()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Between.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Between.java index 2c28c47a85ab33..265b80b60b456c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Between.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Between.java @@ -71,6 +71,11 @@ public String computeToSql() { return compareExpr.toSql() + " BETWEEN " + lowerBound.toSql() + " AND " + upperBound.toSql(); } + @Override + public String shapeInfo() { + return compareExpr.shapeInfo() + " BETWEEN " + lowerBound.shapeInfo() + " AND " + upperBound.shapeInfo(); + } + @Override public String toString() { return compareExpr + " BETWEEN " + lowerBound + " AND " + upperBound; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java index a2cd9cb4730d9c..b7e3c8719d1798 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java @@ -167,6 +167,18 @@ public String computeToSql() throws UnboundException { return output.toString(); } + @Override + public String shapeInfo() { + StringBuilder output = new StringBuilder("CASE"); + value.ifPresent(v -> output.append(" ").append(v.shapeInfo())); + for (WhenClause whenClause : whenClauses) { + output.append(whenClause.shapeInfo()); + } + defaultValue.ifPresent(dv -> output.append(" ELSE ").append(dv.shapeInfo())); + output.append(" END"); + return output.toString(); + } + @Override public CaseWhen withChildren(List children) { Preconditions.checkArgument(!children.isEmpty(), "case when should has at least 1 child"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java index b9d0b48cd81be0..1e14d3284ea768 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java @@ -221,6 +221,11 @@ public String computeToSql() throws UnboundException { return "cast(" + child().toSql() + " as " + targetType.toSql() + ")"; } + @Override + public String shapeInfo() { + return "cast(" + child().shapeInfo() + " as " + targetType.toSql() + ")"; + } + @Override public String toString() { return "cast(" + child() + " as " + targetType + ")"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java index 533f1aa45942eb..b610f4b1a3d3c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java @@ -219,6 +219,13 @@ public String computeToSql() { .collect(Collectors.joining(", ", "(", ")")); } + @Override + public String shapeInfo() { + return compareExpr.shapeInfo() + " IN " + options.stream() + .map(Expression::shapeInfo).sorted() + .collect(Collectors.joining(", ", "(", ")")); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsFalse.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsFalse.java index 09f2a77039fdb6..957bc49f0251dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsFalse.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsFalse.java @@ -60,6 +60,11 @@ public String computeToSql() throws UnboundException { return child().toSql() + " IS FALSE"; } + @Override + public String shapeInfo() { + return child().shapeInfo() + " IS FALSE"; + } + @Override public String toString() { return child().toString() + " IS FALSE"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java index 56371c4c77c893..f1f9b050dccb07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java @@ -59,6 +59,11 @@ public String computeToSql() throws UnboundException { return child().toSql() + " IS NULL"; } + @Override + public String shapeInfo() { + return child().shapeInfo() + " IS NULL"; + } + @Override public String toString() { return child().toString() + " IS NULL"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsTrue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsTrue.java index 1876a6dec54b00..8d34fcb4242bf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsTrue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsTrue.java @@ -60,6 +60,11 @@ public String computeToSql() throws UnboundException { return child().toSql() + " IS TRUE"; } + @Override + public String shapeInfo() { + return child().shapeInfo() + " IS TRUE"; + } + @Override public String toString() { return child().toString() + " IS TRUE"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Like.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Like.java index f979d719f85c86..027938c15972c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Like.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Like.java @@ -69,6 +69,15 @@ public String computeToSql() { + ')'; } + @Override + public String shapeInfo() { + if (arity() == 2) { + return super.shapeInfo(); + } + return '(' + left().shapeInfo() + ' ' + getName() + ' ' + right().shapeInfo() + + " escape " + child(2).shapeInfo() + ')'; + } + @Override public String toString() { if (arity() == 2) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Match.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Match.java index aa2f28563d3ccf..2737ef31eb65b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Match.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Match.java @@ -115,6 +115,12 @@ public String computeToSql() { + analyzerSqlFragment() + ")"; } + @Override + public String shapeInfo() { + return "(" + left().shapeInfo() + " " + symbol + " " + right().shapeInfo() + + analyzerSqlFragment() + ")"; + } + @Override public String toString() { return "(" + left().toString() + " " + symbol + " " + right().toString() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java index e12276ff57fb4d..7c55e913ee01e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Not.java @@ -111,6 +111,11 @@ public String computeToSql() { return "( not " + child().toSql() + ")"; } + @Override + public String shapeInfo() { + return "( not " + child().shapeInfo() + ")"; + } + @Override public Not withChildren(List children) { Preconditions.checkArgument(children.size() == 1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java index dffac46e5b96e1..a596f4bc913511 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/OrderExpression.java @@ -90,6 +90,11 @@ public String computeToSql() { return orderKey.toSql(); } + @Override + public String shapeInfo() { + return orderKey.shapeInfo(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SessionVarGuardExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SessionVarGuardExpr.java index 0c102d6f6aaa5f..7d5cf105f0f2c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SessionVarGuardExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SessionVarGuardExpr.java @@ -72,6 +72,13 @@ public String computeToSql() { } } + @Override + public String shapeInfo() { + try (AutoCloseSessionVariable ignored = openGuard()) { + return child().shapeInfo(); + } + } + @Override public Expression withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "SessionVarGuardExpr must have exactly one child"); @@ -144,4 +151,3 @@ public static List getExprWithGuard(Map children) { Preconditions.checkArgument(children.size() == 2); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java index 830b807e6ef28a..0921fa5028e3fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowExpression.java @@ -207,6 +207,24 @@ public String computeToSql() { return sb.toString().trim() + ")"; } + @Override + public String shapeInfo() { + StringBuilder sb = new StringBuilder(); + sb.append(function.shapeInfo()).append(" OVER("); + if (!partitionKeys.isEmpty()) { + sb.append("PARTITION BY ").append(partitionKeys.stream() + .map(Expression::shapeInfo) + .collect(Collectors.joining(", ", "", " "))); + } + if (!orderKeys.isEmpty()) { + sb.append("ORDER BY ").append(orderKeys.stream() + .map(OrderExpression::shapeInfo) + .collect(Collectors.joining(", ", "", " "))); + } + windowFrame.ifPresent(wf -> sb.append(wf.toSql())); + return sb.toString().trim() + ")"; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java index 3212d8ff24db60..bbbaede7159e2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/BoundFunction.java @@ -99,6 +99,20 @@ public String computeToSql() throws UnboundException { return sql.append(")").toString(); } + @Override + public String shapeInfo() { + StringBuilder sql = new StringBuilder(getName()).append("("); + int arity = arity(); + for (int i = 0; i < arity; i++) { + Expression arg = child(i); + sql.append(arg.shapeInfo()); + if (i + 1 < arity) { + sql.append(", "); + } + } + return sql.append(")").toString(); + } + @Override public String toString() { String args = children() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java index 380772afb5543b..3251e63c4bb5ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java @@ -153,6 +153,22 @@ public String computeToSql() throws UnboundException { return sql.append(")").toString(); } + @Override + public String shapeInfo() { + StringBuilder sql = new StringBuilder(getName()).append("("); + if (distinct) { + sql.append("DISTINCT "); + } + int arity = arity(); + for (int i = 0; i < arity; i++) { + sql.append(child(i).shapeInfo()); + if (i + 1 < arity) { + sql.append(", "); + } + } + return sql.append(")").toString(); + } + @Override public String toString() { String args = children() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java index e6169edd45ce33..6347bec6dc096e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java @@ -563,7 +563,7 @@ private static int compareFloatLiteral(FloatLiteral first, FloatLiteral... secon float firstValue = first.getValue(); for (int i = 0; i < second.length; i++) { float secondValue = second[i].getValue(); - if (secondValue == firstValue) { + if (secondValue == firstValue || Float.isNaN(secondValue) && Float.isNaN(firstValue)) { return i + 1; } } @@ -574,7 +574,7 @@ private static int compareDoubleLiteral(DoubleLiteral first, DoubleLiteral... se double firstValue = first.getValue(); for (int i = 0; i < second.length; i++) { double secondValue = second[i].getValue(); - if (secondValue == firstValue) { + if (secondValue == firstValue || Double.isNaN(secondValue) && Double.isNaN(firstValue)) { return i + 1; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonExtractNoQuotes.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonExtractNoQuotes.java deleted file mode 100644 index 02efc44c964a12..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/JsonExtractNoQuotes.java +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; -import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.JsonType; -import org.apache.doris.nereids.types.VarcharType; -import org.apache.doris.nereids.util.ExpressionUtils; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * ScalarFunction 'json_object'. - */ -public class JsonExtractNoQuotes extends ScalarFunction - implements ExplicitlyCastableSignature, AlwaysNullable { - - public static final List SIGNATURES = - ImmutableList.of(FunctionSignature.ret(JsonType.INSTANCE) - .varArgs(JsonType.INSTANCE, VarcharType.SYSTEM_DEFAULT)); - - /** - * constructor with 1 or more arguments. - */ - public JsonExtractNoQuotes(Expression arg0, Expression arg1, Expression... varArgs) { - super("json_extract_no_quotes", ExpressionUtils.mergeArguments(arg0, arg1, varArgs)); - } - - /** constructor for withChildren and reuse signature */ - private JsonExtractNoQuotes(ScalarFunctionParams functionParams) { - super(functionParams); - } - - /** - * withChildren. - */ - @Override - public JsonExtractNoQuotes withChildren(List children) { - Preconditions.checkArgument(children.size() >= 2); - return new JsonExtractNoQuotes(getFunctionParams(children)); - } - - @Override - public List getSignatures() { - return SIGNATURES; - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitJsonExtractNoQuotes(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java index 6436f439ce52b4..31887127eb4556 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java @@ -19,10 +19,13 @@ import org.apache.doris.analysis.DateLiteral; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.ExpressionEvaluator; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -90,12 +93,12 @@ public FunctionSignature computeSignature(FunctionSignature signature) { * Return type is DATETIME */ DataType returnType; - if (getArgument(1) instanceof StringLikeLiteral) { - if (DateLiteral.hasTimePart(((StringLikeLiteral) getArgument(1)).getStringValue())) { + Literal formatLiteral = getConstantFormatLiteral(); + if (formatLiteral != null) { + if (DateLiteral.hasTimePart(formatLiteral.getStringValue())) { //FIXME: Here will pass different scale to BE with same input types. Need to be fixed. returnType = DateTimeV2Type.SYSTEM_DEFAULT; - if (returnType.isDateTimeV2Type() - && DateLiteral.hasMicroSecondPart(((StringLikeLiteral) getArgument(1)).getStringValue())) { + if (DateLiteral.hasMicroSecondPart(formatLiteral.getStringValue())) { returnType = DateTimeV2Type.MAX; } } else { @@ -107,6 +110,24 @@ public FunctionSignature computeSignature(FunctionSignature signature) { return signature.withReturnType(returnType); } + private StringLikeLiteral getConstantFormatLiteral() { + Expression format = getArgument(1); + if (!format.isConstant()) { + return null; + } + if (!format.getDataType().isStringLikeType()) { + format = new Cast(format, StringType.INSTANCE); + } + if (format instanceof StringLikeLiteral) { + return (StringLikeLiteral) format; + } + Expression evaluated = ExpressionEvaluator.INSTANCE.eval(format); + if (evaluated instanceof StringLikeLiteral) { + return (StringLikeLiteral) evaluated; + } + return null; + } + /** * withChildren. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index df42c3709497b4..3d1f5017be3b14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -370,7 +370,7 @@ protected void init(String s) throws AnalysisException { month = DateUtils.getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR); day = DateUtils.getOrDefault(dateTime, ChronoField.DAY_OF_MONTH); - if (checkDatetime(dateTime) || checkRange(year, month, day) || checkDate(year, month, day)) { + if (checkRange(year, month, day) || checkDate(year, month, day)) { throw new AnalysisException("date/datetime literal [" + s + "] is out of range"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index 4a8ac991168f81..c50f98b45a1119 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -246,27 +246,16 @@ protected void init(String s) throws AnalysisException { // Microseconds have 7 digits. long sevenDigit = microSecond % 10; microSecond = microSecond / 10; - if (sevenDigit >= 5 && (this instanceof DateTimeV2Literal || this instanceof TimestampTzLiteral)) { + if (sevenDigit >= 5) { DateTimeLiteral result; - if (this instanceof DateTimeV2Literal) { - result = (DateTimeV2Literal) ((DateTimeV2Literal) this).plusMicroSeconds(1); - this.second = result.second; - this.minute = result.minute; - this.hour = result.hour; - this.day = result.day; - this.month = result.month; - this.year = result.year; - this.microSecond = result.microSecond; - } else if (this instanceof TimestampTzLiteral) { - result = (TimestampTzLiteral) ((TimestampTzLiteral) this).plusMicroSeconds(1); - this.second = result.second; - this.minute = result.minute; - this.hour = result.hour; - this.day = result.day; - this.month = result.month; - this.year = result.year; - this.microSecond = result.microSecond; - } + result = this.plusMicroSeconds(1); + this.second = result.second; + this.minute = result.minute; + this.hour = result.hour; + this.day = result.day; + this.month = result.month; + this.year = result.year; + this.microSecond = result.microSecond; } if (checkRange(year, month, day) || checkDate(year, month, day)) { @@ -274,6 +263,12 @@ protected void init(String s) throws AnalysisException { } } + // When performing addition or subtraction with MicroSeconds, the precision must be set to 6 to display it + // completely. use multiplyExact to be aware of multiplication overflow possibility. + public DateTimeLiteral plusMicroSeconds(long microSeconds) { + return fromJavaDateType(toJavaDateType().plusNanos(Math.multiplyExact(microSeconds, 1000L)), 6); + } + private static LocalDateTime convertTimeZone(long year, long month, long day, long hour, long minute, long second, ZoneId fromZone, ZoneId toZone) { LocalDateTime localDateTime = LocalDateTime.of((int) year, (int) month, (int) day, @@ -438,31 +433,31 @@ public LiteralExpr toLegacyLiteral() { } public Expression plusDays(long days) { - return fromJavaDateType(toJavaDateType().plusDays(days)); + return fromJavaDateType(toJavaDateType().plusDays(days), 0); } public Expression plusMonths(long months) { - return fromJavaDateType(toJavaDateType().plusMonths(months)); + return fromJavaDateType(toJavaDateType().plusMonths(months), 0); } public Expression plusWeeks(long weeks) { - return fromJavaDateType(toJavaDateType().plusWeeks(weeks)); + return fromJavaDateType(toJavaDateType().plusWeeks(weeks), 0); } public Expression plusYears(long years) { - return fromJavaDateType(toJavaDateType().plusYears(years)); + return fromJavaDateType(toJavaDateType().plusYears(years), 0); } public Expression plusHours(long hours) { - return fromJavaDateType(toJavaDateType().plusHours(hours)); + return fromJavaDateType(toJavaDateType().plusHours(hours), 0); } public Expression plusMinutes(long minutes) { - return fromJavaDateType(toJavaDateType().plusMinutes(minutes)); + return fromJavaDateType(toJavaDateType().plusMinutes(minutes), 0); } public Expression plusSeconds(long seconds) { - return fromJavaDateType(toJavaDateType().plusSeconds(seconds)); + return fromJavaDateType(toJavaDateType().plusSeconds(seconds), 0); } public long getHour() { @@ -498,7 +493,7 @@ public LocalDateTime toJavaDateType() { ((int) getHour()), ((int) getMinute()), ((int) getSecond()), (int) getMicroSecond() * 1000); } - public static Expression fromJavaDateType(LocalDateTime dateTime) { + public static DateTimeLiteral fromJavaDateType(LocalDateTime dateTime, int precision) { if (isDateOutOfRange(dateTime)) { throw new AnalysisException("datetime out of range: " + dateTime.toString()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java index 93f221d4d99cc0..bfdf1e3be6eba5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java @@ -409,7 +409,7 @@ public Expression plusSecondMicrosecond(VarcharLiteral secondMicrosecond) { // When performing addition or subtraction with MicroSeconds, the precision must be set to 6 to display it // completely. use multiplyExact to be aware of multiplication overflow possibility. - public Expression plusMicroSeconds(long microSeconds) { + public DateTimeV2Literal plusMicroSeconds(long microSeconds) { return fromJavaDateType(toJavaDateType().plusNanos(Math.multiplyExact(microSeconds, 1000L)), 6); } @@ -474,7 +474,7 @@ public static Expression fromJavaDateType(LocalDateTime dateTime) { /** * convert java LocalDateTime object to DateTimeV2Literal object. */ - public static Expression fromJavaDateType(LocalDateTime dateTime, int precision) { + public static DateTimeV2Literal fromJavaDateType(LocalDateTime dateTime, int precision) { long value = (long) Math.pow(10, DateTimeV2Type.MAX_SCALE - precision); if (isDateOutOfRange(dateTime)) { throw new AnalysisException("datetime out of range" + dateTime.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java index c8c161b35bdc79..e82acd634c7f69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java @@ -174,7 +174,12 @@ public boolean equals(Object o) { @Override public String computeToSql() { - return value.toPlainString(); + return getStringValue(); + } + + @Override + public String getStringValue() { + return value.setScale(((DecimalV3Type) dataType).getScale(), RoundingMode.HALF_UP).toPlainString(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java index 148c3efb8529be..afe473ccf75ca7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.format.DateTimeChecker; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; @@ -133,18 +132,9 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti return castToIntegral(targetType, strictCast); } if (targetType.isDateType() || targetType.isDateV2Type()) { - Expression expression = castToDateTime(DateTimeV2Type.MAX, strictCast, false); - DateTimeV2Literal datetime = (DateTimeV2Literal) expression; - if (targetType.isDateType()) { - return new DateLiteral(datetime.year, datetime.month, datetime.day); - } else { - return new DateV2Literal(datetime.year, datetime.month, datetime.day); - } + return castToDateTime(targetType, strictCast); } else if (targetType.isDateTimeType()) { - Expression expression = castToDateTime(DateTimeV2Type.MAX, strictCast, true); - DateTimeV2Literal datetime = (DateTimeV2Literal) expression; - return new DateTimeLiteral((DateTimeType) targetType, datetime.year, datetime.month, datetime.day, - datetime.hour, datetime.minute, datetime.second, datetime.microSecond); + return castToDateTime(targetType, strictCast); } else if (targetType.isTimeStampTzType()) { // Wildcard targets still need a concrete scale before parsing. TimeStampTzType timeStampTzType = (TimeStampTzType) targetType; @@ -154,10 +144,10 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti if (DateTimeChecker.hasTimeZone(value)) { return new TimestampTzLiteral(timeStampTzType, value); } - DateTimeV2Literal datetime = castToDateTime(DateTimeV2Type.MAX, strictCast, true); + DateTimeV2Literal datetime = (DateTimeV2Literal) castToDateTime(DateTimeV2Type.MAX, strictCast); return TimestampTzLiteral.fromSessionTimeZone(timeStampTzType, datetime); } else if (targetType.isDateTimeV2Type()) { - return castToDateTime(targetType, strictCast, true); + return castToDateTime(targetType, strictCast); } else if (targetType.isFloatType()) { return castToFloat(); } else if (targetType.isDoubleType()) { @@ -270,7 +260,7 @@ protected Expression castToDecimal(DataType targetType) { throw new CastException(String.format("%s can't cast to decimal in strict mode.", value)); } - protected DateTimeV2Literal castToDateTime(DataType targetType, boolean strictCast, boolean isDatetime) { + protected DateLiteral castToDateTime(DataType targetType, boolean strictCast) { Matcher strictMatcher = dateStrictPattern.matcher(value); String year; String month; @@ -312,14 +302,7 @@ protected DateTimeV2Literal castToDateTime(DataType targetType, boolean strictCa if (tz != null && tz.equalsIgnoreCase("CST")) { tz = "+08:00"; } - DateTimeV2Literal dt = getDateTimeLiteral(year, month, date, hour, minute, second, - fraction, tz, targetType); - if (isDatetime) { - return dt; - } else { - return new DateTimeV2Literal(Long.parseLong(year2ToYear4(year)), Long.parseLong(month), - Long.parseLong(date), 0, 0, 0); - } + return getDateTimeLiteral(year, month, date, hour, minute, second, fraction, tz, targetType); } else if (!strictCast) { Matcher unStrictMatcher = dateUnStrictPattern.matcher(value); if (unStrictMatcher.matches()) { @@ -334,14 +317,7 @@ protected DateTimeV2Literal castToDateTime(DataType targetType, boolean strictCa if (tz != null && tz.equalsIgnoreCase("CST")) { tz = "+08:00"; } - DateTimeV2Literal dt = getDateTimeLiteral(year, month, date, hour, minute, second, - fraction, tz, targetType); - if (isDatetime) { - return dt; - } else { - return new DateTimeV2Literal(Long.parseLong(year2ToYear4(year)), Long.parseLong(month), - Long.parseLong(date), 0, 0, 0); - } + return getDateTimeLiteral(year, month, date, hour, minute, second, fraction, tz, targetType); } } throw new CastException(String.format("[%s] can't cast to %s.", value, targetType)); @@ -355,7 +331,7 @@ protected String year2ToYear4(String year) { return year; } - protected DateTimeV2Literal getDateTimeLiteral(String year, String month, String date, String hour, String minute, + protected DateLiteral getDateTimeLiteral(String year, String month, String date, String hour, String minute, String second, String fraction, String tz, DataType targetType) { String year4 = year2ToYear4(year); tz = tz == null ? "" : tz; @@ -385,10 +361,30 @@ protected DateTimeV2Literal getDateTimeLiteral(String year, String month, String } } String format = String.format("%s-%s-%sT%s:%s:%s%s%s", year4, month, date, hour, minute, second, fraction, tz); - try { - return new DateTimeV2Literal((DateTimeV2Type) targetType, format); - } catch (AnalysisException e) { - throw new CastException(e.getMessage(), e); + if (targetType.isDateType()) { + try { + return new DateLiteral(format); + } catch (AnalysisException e) { + throw new CastException(e.getMessage(), e); + } + } else if (targetType.isDateV2Type()) { + try { + return new DateV2Literal(format); + } catch (AnalysisException e) { + throw new CastException(e.getMessage(), e); + } + } else if (targetType.isDateTimeType()) { + try { + return new DateTimeLiteral(format); + } catch (AnalysisException e) { + throw new CastException(e.getMessage(), e); + } + } else { + try { + return new DateTimeV2Literal((DateTimeV2Type) targetType, format); + } catch (AnalysisException e) { + throw new CastException(e.getMessage(), e); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java index 0b2c7f067b635d..8d04a006baf51a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java @@ -302,9 +302,10 @@ public int getMicroSecond() { @Override protected Expression uncheckedCastTo(DataType targetType) throws AnalysisException { + long microsecondValue = ((Double) getValue()).longValue(); DateTimeV2Literal time = (DateTimeV2Literal) DateTimeV2Literal.fromJavaDateType(LocalDateTime - .now(DateUtils.getTimeZone()).withHour(0).withMinute(0).withSecond(0).withNano(0).plusHours(getHour()) - .plusMinutes(getMinute()).plusSeconds(getSecond()).plusNanos(getMicroSecond() * 1000), + .now(DateUtils.getTimeZone()).withHour(0).withMinute(0).withSecond(0).withNano(0) + .plusNanos(microsecondValue * 1000), ((TimeV2Type) dataType).getScale()); if (targetType.isDateType()) { return new DateLiteral(time.getYear(), time.getMonth(), time.getDay()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java index 7caed5ecc01b46..810168d4985a51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java @@ -380,7 +380,7 @@ public Expression plusSeconds(long seconds) { // When performing addition or subtraction with MicroSeconds, the precision must be set to 6 to display it // completely. use multiplyExact to be aware of multiplication overflow possibility. - public Expression plusMicroSeconds(long microSeconds) { + public TimestampTzLiteral plusMicroSeconds(long microSeconds) { return fromJavaDateType(toJavaDateType().plusNanos(Math.multiplyExact(microSeconds, 1000L)), 6); } @@ -473,7 +473,7 @@ public static Expression fromJavaDateType(LocalDateTime dateTime) { /** * convert java LocalDateTime object to TimeStampTzTypeLiteral object. */ - public static Expression fromJavaDateType(LocalDateTime dateTime, int precision) { + public static TimestampTzLiteral fromJavaDateType(LocalDateTime dateTime, int precision) { long value = (long) Math.pow(10, TimeStampTzType.MAX_SCALE - precision); if (isDateOutOfRange(dateTime)) { throw new AnalysisException("datetime out of range" + dateTime.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 68afa81f4fd929..7b1724ae085497 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -305,7 +305,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonArray; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonArrayIgnoreNull; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonContains; -import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonExtractNoQuotes; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonHash; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonInsert; import org.apache.doris.nereids.trees.expressions.functions.scalar.JsonKeys; @@ -1748,10 +1747,6 @@ default R visitJsonObjectFlatten(JsonObjectFlatten jsonObjectFlatten, C context) return visitScalarFunction(jsonObjectFlatten, context); } - default R visitJsonExtractNoQuotes(JsonExtractNoQuotes jsonExtract, C context) { - return visitScalarFunction(jsonExtract, context); - } - default R visitJsonHash(JsonHash jsonhash, C context) { return visitScalarFunction(jsonhash, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterComputeGroupCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterComputeGroupCommand.java index 8a295678f4826e..d5f520661b8f8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterComputeGroupCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterComputeGroupCommand.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.catalog.Env; -import org.apache.doris.cloud.catalog.ComputeGroup; +import org.apache.doris.cloud.catalog.CloudComputeGroupMeta; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -68,7 +68,7 @@ public void validate(ConnectContext connectContext) throws UserException { CloudSystemInfoService cloudSys = ((CloudSystemInfoService) Env.getCurrentSystemInfo()); // check compute group exist - ComputeGroup cg = cloudSys.getComputeGroupByName(computeGroupName); + CloudComputeGroupMeta cg = cloudSys.getComputeGroupByName(computeGroupName); if (cg == null) { throw new AnalysisException("Compute Group " + computeGroupName + " does not exist"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index 86b085740e2925..91fb9bf9739019 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.analysis.StmtType; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; @@ -36,6 +37,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.InternalDatabaseUtil; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp; @@ -52,6 +54,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.DropRollupOp; import org.apache.doris.nereids.trees.plans.commands.info.DropTagOp; import org.apache.doris.nereids.trees.plans.commands.info.EnableFeatureOp; +import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnCommentOp; import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnOp; import org.apache.doris.nereids.trees.plans.commands.info.ModifyEngineOp; import org.apache.doris.nereids.trees.plans.commands.info.ModifyTablePropertiesOp; @@ -115,10 +118,6 @@ private void validate(ConnectContext ctx) throws UserException { if (ops == null || ops.isEmpty()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_ALTER_OPERATION); } - for (AlterTableOp op : ops) { - op.setTableName(tbl); - op.validate(ctx); - } String ctlName = tbl.getCtl(); String dbName = tbl.getDb(); String tableName = tbl.getTbl(); @@ -129,6 +128,11 @@ private void validate(ConnectContext ctx) throws UserException { if (tableIf.isTemporary()) { throw new AnalysisException("Do not support alter temporary table[" + tableName + "]"); } + checkColumnOperationsSupported(tableIf, ops); + for (AlterTableOp op : ops) { + op.setTableName(tbl); + op.validate(ctx); + } if (tableIf instanceof OlapTable) { rewriteAlterOpForOlapTable(ctx, (OlapTable) tableIf); } else { @@ -136,6 +140,140 @@ private void validate(ConnectContext ctx) throws UserException { } } + static void checkColumnOperationsSupported(TableIf table, List alterTableOps) + throws AnalysisException { + if (table instanceof IcebergExternalTable) { + checkIcebergCompoundColumnOperations(alterTableOps); + for (AlterTableOp alterTableOp : alterTableOps) { + ColumnDefinition columnDefinition = getColumnDefinition(alterTableOp); + ColumnPath nestedColumnPath = getNestedColumnPath(alterTableOp); + // Keep this before AddColumnOp.validate(), whose generic NOT NULL check would mask + // the Iceberg nested-field invariant. Metadata validation retains the same guard for other callers. + if (alterTableOp instanceof AddColumnOp && columnDefinition != null && nestedColumnPath != null + && !columnDefinition.isNullable()) { + throw new AnalysisException("New nested field '" + nestedColumnPath.getFullPath() + + "' must be nullable"); + } + if (!isIcebergColumnSchemaOperation(alterTableOp)) { + continue; + } + if (getRollupName(alterTableOp) != null) { + throw new AnalysisException("Rollup is not supported for Iceberg column operations"); + } + Map properties = alterTableOp.getProperties(); + if (properties != null && !properties.isEmpty()) { + throw new AnalysisException("PROPERTIES are not supported for Iceberg column operations"); + } + checkIcebergColumnDefinition(alterTableOp, columnDefinition); + if (alterTableOp instanceof AddColumnsOp) { + for (ColumnDefinition definition : ((AddColumnsOp) alterTableOp).getColumnDefinitions()) { + checkIcebergColumnDefinition(alterTableOp, definition); + } + } + } + return; + } + for (AlterTableOp alterTableOp : alterTableOps) { + ColumnPath columnPath = getNestedColumnPath(alterTableOp); + if (columnPath != null) { + throw new AnalysisException("Nested column path is only supported for Iceberg tables: " + + columnPath.getFullPath()); + } + } + } + + private static void checkIcebergCompoundColumnOperations(List alterTableOps) + throws AnalysisException { + if (alterTableOps.size() <= 1) { + return; + } + for (AlterTableOp alterTableOp : alterTableOps) { + if (isIcebergColumnSchemaOperation(alterTableOp)) { + throw new AnalysisException("Multiple Iceberg ALTER clauses are not supported when a statement " + + "contains a column operation"); + } + } + } + + private static ColumnPath getNestedColumnPath(AlterTableOp alterTableOp) { + ColumnPath columnPath = null; + if (alterTableOp instanceof AddColumnOp) { + columnPath = ((AddColumnOp) alterTableOp).getColumnPath(); + } else if (alterTableOp instanceof DropColumnOp) { + columnPath = ((DropColumnOp) alterTableOp).getColumnPath(); + } else if (alterTableOp instanceof RenameColumnOp) { + columnPath = ((RenameColumnOp) alterTableOp).getColumnPath(); + } else if (alterTableOp instanceof ModifyColumnOp) { + columnPath = ((ModifyColumnOp) alterTableOp).getColumnPath(); + } else if (alterTableOp instanceof ModifyColumnCommentOp) { + columnPath = ((ModifyColumnCommentOp) alterTableOp).getColumnPath(); + } + return columnPath != null && columnPath.isNested() ? columnPath : null; + } + + private static ColumnDefinition getColumnDefinition(AlterTableOp alterTableOp) { + if (alterTableOp instanceof AddColumnOp) { + return ((AddColumnOp) alterTableOp).getColumnDef(); + } + if (alterTableOp instanceof ModifyColumnOp) { + return ((ModifyColumnOp) alterTableOp).getColumnDef(); + } + return null; + } + + private static boolean isIcebergColumnSchemaOperation(AlterTableOp alterTableOp) { + return alterTableOp instanceof AddColumnOp + || alterTableOp instanceof AddColumnsOp + || alterTableOp instanceof DropColumnOp + || alterTableOp instanceof RenameColumnOp + || alterTableOp instanceof ModifyColumnOp + || alterTableOp instanceof ModifyColumnCommentOp + || alterTableOp instanceof ReorderColumnsOp; + } + + private static void checkIcebergColumnDefinition(AlterTableOp alterTableOp, ColumnDefinition columnDefinition) + throws AnalysisException { + if (columnDefinition == null) { + return; + } + if (columnDefinition.isKey()) { + throw new AnalysisException("KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + } + if (columnDefinition.getGeneratedColumnDesc().isPresent()) { + throw new AnalysisException("Generated columns are not supported for Iceberg ADD/MODIFY COLUMN"); + } + if (alterTableOp instanceof ModifyColumnOp + && (columnDefinition.hasDefaultValue() || columnDefinition.hasOnUpdateDefaultValue())) { + columnDefinition.validateComplexTypeDefaultValue(); + throw new AnalysisException("Modifying default values is not supported for Iceberg columns: " + + ((ModifyColumnOp) alterTableOp).getColumnPath().getFullPath()); + } + if ((alterTableOp instanceof AddColumnOp || alterTableOp instanceof AddColumnsOp) + && columnDefinition.hasOnUpdateDefaultValue()) { + throw new AnalysisException("ON UPDATE is not supported for Iceberg ADD COLUMN: " + + columnDefinition.getName()); + } + } + + private static String getRollupName(AlterTableOp alterTableOp) { + if (alterTableOp instanceof AddColumnOp) { + return ((AddColumnOp) alterTableOp).getRollupName(); + } + if (alterTableOp instanceof AddColumnsOp) { + return ((AddColumnsOp) alterTableOp).getRollupName(); + } + if (alterTableOp instanceof DropColumnOp) { + return ((DropColumnOp) alterTableOp).getRollupName(); + } + if (alterTableOp instanceof ModifyColumnOp) { + return ((ModifyColumnOp) alterTableOp).getRollupName(); + } + if (alterTableOp instanceof ReorderColumnsOp) { + return ((ReorderColumnsOp) alterTableOp).getRollupName(); + } + return null; + } + private void rewriteAlterOpForOlapTable(ConnectContext ctx, OlapTable table) throws UserException { List alterTableOps = new ArrayList<>(); for (AlterTableOp alterClause : ops) { @@ -242,6 +380,7 @@ private void checkExternalTableOperationAllow(TableIf table) throws UserExceptio || alterTableOp instanceof DropColumnOp || alterTableOp instanceof RenameColumnOp || alterTableOp instanceof ModifyColumnOp + || (alterTableOp instanceof ModifyColumnCommentOp && table instanceof IcebergExternalTable) || alterTableOp instanceof ReorderColumnsOp || alterTableOp instanceof ModifyEngineOp || alterTableOp instanceof ModifyTablePropertiesOp diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java index d939d278ddeff5..67a0a81932df2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java @@ -212,7 +212,8 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { return; } catch (Exception e2) { LOG.warn("delete from command failed", e2); - throw e; + // Preserve both failure causes so the fallback execution error is not masked. + throw buildDeleteFallbackException(e, e2); } } @@ -285,6 +286,21 @@ private void updateSessionVariableForDelete(SessionVariable sessionVariable) { } } + // Build an exception that keeps both the initial predicate-check failure and the fallback failure. + private AnalysisException buildDeleteFallbackException(Exception initialException, + Exception fallbackException) { + String initialMessage = StringUtils.defaultIfBlank( + initialException.getMessage(), initialException.toString()); + String fallbackMessage = StringUtils.defaultIfBlank( + fallbackException.getMessage(), fallbackException.toString()); + AnalysisException mergedException = new AnalysisException( + "Delete fallback execution failed: " + fallbackMessage + + ". Initial predicate check failed: " + initialMessage, + fallbackException); + mergedException.addSuppressed(initialException); + return mergedException; + } + private List getSelectedPartitions( OlapTable olapTable, PhysicalFilter filter, PhysicalOlapScan scan, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowClustersCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowClustersCommand.java index a4c389941e009d..c774849ed48af4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowClustersCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowClustersCommand.java @@ -22,7 +22,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ScalarType; -import org.apache.doris.cloud.catalog.ComputeGroup; +import org.apache.doris.cloud.catalog.CloudComputeGroupMeta; import org.apache.doris.cloud.qe.ComputeGroupException; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; @@ -96,9 +96,9 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc CloudSystemInfoService cloudSys = ((CloudSystemInfoService) Env.getCurrentSystemInfo()); clusterNames = cloudSys.getCloudClusterNames(); // virtual cluster info - List virtualComputeGroup = cloudSys.getComputeGroups(true); + List virtualComputeGroup = cloudSys.getComputeGroups(true); List virtualComputeGroupNames = virtualComputeGroup.stream() - .map(ComputeGroup::getName).collect(Collectors.toList()); + .map(CloudComputeGroupMeta::getName).collect(Collectors.toList()); clusterNames.addAll(virtualComputeGroupNames); @@ -112,7 +112,7 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)) { continue; } - ComputeGroup cg = cloudSys.getComputeGroupByName(clusterName); + CloudComputeGroupMeta cg = cloudSys.getComputeGroupByName(clusterName); if (cg == null) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowRoutineLoadCommand.java index fe5c1441ebc6d6..42737314470671 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowRoutineLoadCommand.java @@ -110,6 +110,7 @@ public class ShowRoutineLoadCommand extends ShowCommand { .add("User") .add("Comment") .add("ComputeGroup") + .add("FirstErrorMsg") .build(); private final LabelNameInfo labelNameInfo; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/WarmUpClusterCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/WarmUpClusterCommand.java index 4c6d74c898f54f..fd77fb41779de4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/WarmUpClusterCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/WarmUpClusterCommand.java @@ -24,8 +24,8 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.cloud.OnTablesFilter.TableFilterRule; +import org.apache.doris.cloud.catalog.CloudComputeGroupMeta; import org.apache.doris.cloud.catalog.CloudEnv; -import org.apache.doris.cloud.catalog.ComputeGroup; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -141,7 +141,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { private void checkWarmupCgs(CloudSystemInfoService cloudSys) throws AnalysisException { if (!Strings.isNullOrEmpty(srcCluster)) { - ComputeGroup srcCg = cloudSys.getComputeGroupByName(srcCluster); + CloudComputeGroupMeta srcCg = cloudSys.getComputeGroupByName(srcCluster); if (srcCg != null && srcCg.isVirtual()) { throw new AnalysisException("The srcClusterName " + srcCluster + " is a virtual compute group, not support"); @@ -149,7 +149,7 @@ private void checkWarmupCgs(CloudSystemInfoService cloudSys) throws AnalysisExce } if (!Strings.isNullOrEmpty(dstCluster)) { - ComputeGroup dstCg = cloudSys.getComputeGroupByName(dstCluster); + CloudComputeGroupMeta dstCg = cloudSys.getComputeGroupByName(dstCluster); if (dstCg != null && dstCg.isVirtual()) { throw new AnalysisException("The dstClusterName " + dstCluster + " is a virtual compute group, not support"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java index 09c686779a862c..2e6c5ab3727820 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnOp.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -44,6 +45,7 @@ */ public class AddColumnOp extends AlterTableOp { private ColumnDefinition columnDef; + private ColumnPath columnPath; // Column position private ColumnPosition colPos; // if rollupName is null, add to column to base index. @@ -56,8 +58,17 @@ public class AddColumnOp extends AlterTableOp { public AddColumnOp(ColumnDefinition columnDef, ColumnPosition colPos, String rollupName, Map properties) { + this(columnDef, ColumnPath.of(columnDef.getName()), colPos, rollupName, properties); + } + + /** + * Create add-column operation with the original nested column path. + */ + public AddColumnOp(ColumnDefinition columnDef, ColumnPath columnPath, ColumnPosition colPos, + String rollupName, Map properties) { super(AlterOpType.SCHEMA_CHANGE); this.columnDef = columnDef; + this.columnPath = columnPath; this.colPos = colPos; this.rollupName = rollupName; this.properties = properties; @@ -67,6 +78,14 @@ public Column getColumn() { return column; } + public ColumnDefinition getColumnDef() { + return columnDef; + } + + public ColumnPath getColumnPath() { + return columnPath; + } + public ColumnPosition getColPos() { return colPos; } @@ -84,7 +103,7 @@ public void validate(ConnectContext ctx) throws UserException { if (colPos != null) { colPos.analyze(); } - validateColumnDef(tableName, columnDef, colPos, rollupName); + validateColumnDef(tableName, columnDef, colPos, rollupName, columnPath.isNested()); column = columnDef.translateToCatalogStyleForSchemaChange(); } @@ -111,7 +130,7 @@ public boolean allowOpRowBinlog() { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("ADD COLUMN ").append(columnDef.toSql()); + sb.append("ADD COLUMN ").append(columnDef.toSql(columnPath.toSql())); if (colPos != null) { sb.append(" ").append(colPos.toSql()); } @@ -132,6 +151,11 @@ public String toString() { public static void validateColumnDef(TableNameInfo tableName, ColumnDefinition columnDef, ColumnPosition colPos, String rollupName) throws UserException { + validateColumnDef(tableName, columnDef, colPos, rollupName, false); + } + + private static void validateColumnDef(TableNameInfo tableName, ColumnDefinition columnDef, ColumnPosition colPos, + String rollupName, boolean nestedColumn) throws UserException { if (columnDef == null) { throw new AnalysisException("No column definition in add column clause."); } @@ -182,7 +206,11 @@ public static void validateColumnDef(TableNameInfo tableName, ColumnDefinition c } } - columnDef.validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + if (nestedColumn) { + columnDef.validateNestedColumn(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + } else { + columnDef.validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + } if (!columnDef.isNullable() && !columnDef.hasDefaultValue()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DEFAULT_FOR_FIELD, columnDef.getName()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnsOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnsOp.java index 418185ed7557f2..9ff326ffb385de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnsOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AddColumnsOp.java @@ -62,6 +62,10 @@ public List getColumns() { return columns; } + public List getColumnDefinitions() { + return columnDefs == null ? Collections.emptyList() : columnDefs; + } + public String getRollupName() { return rollupName; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 4be6315f079fdb..6c386489139492 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.CharacterType; +import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContextUtil; import org.apache.doris.qe.SessionVariable; @@ -61,9 +62,13 @@ public class ColumnDefinition { private boolean isKey; private AggregateType aggType; private boolean isNullable; + // Distinguishes an explicit NULL/NOT NULL clause from the parser's default nullability. + private final boolean nullableSpecified; private Optional defaultValue; private Optional onUpdateDefaultValue = Optional.empty(); private final String comment; + // Distinguishes an explicit COMMENT '' clause from an omitted COMMENT clause. + private final boolean commentSpecified; private final boolean isVisible; private boolean aggTypeImplicit = false; private long autoIncInitValue = -1; @@ -83,7 +88,7 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType Optional onUpdateDefaultValue, String comment, Optional generatedColumnDesc) { this(name, type, isKey, aggType, nullableType, autoIncInitValue, defaultValue, onUpdateDefaultValue, - comment, true, generatedColumnDesc); + comment, comment != null && !comment.isEmpty(), true, generatedColumnDesc); } /** @@ -96,8 +101,10 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType this.isKey = isKey; this.aggType = aggType; this.isNullable = isNullable; + this.nullableSpecified = true; this.defaultValue = defaultValue; this.comment = comment; + this.commentSpecified = comment != null && !comment.isEmpty(); this.isVisible = isVisible; } @@ -112,10 +119,12 @@ private ColumnDefinition(String name, DataType type, boolean isKey, AggregateTyp this.isKey = isKey; this.aggType = aggType; this.isNullable = isNullable; + this.nullableSpecified = true; this.autoIncInitValue = autoIncInitValue; this.defaultValue = defaultValue; this.onUpdateDefaultValue = onUpdateDefaultValue; this.comment = comment; + this.commentSpecified = comment != null && !comment.isEmpty(); this.isVisible = isVisible; } @@ -126,15 +135,30 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType ColumnNullableType nullableType, long autoIncInitValue, Optional defaultValue, Optional onUpdateDefaultValue, String comment, boolean isVisible, Optional generatedColumnDesc) { + this(name, type, isKey, aggType, nullableType, autoIncInitValue, defaultValue, onUpdateDefaultValue, + comment, comment != null && !comment.isEmpty(), isVisible, generatedColumnDesc); + } + + /** + * constructor + */ + public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, + ColumnNullableType nullableType, long autoIncInitValue, Optional defaultValue, + Optional onUpdateDefaultValue, String comment, boolean commentSpecified, + boolean isVisible, + Optional generatedColumnDesc) { this.name = name; this.type = type; this.isKey = isKey; this.aggType = aggType; this.isNullable = nullableType.getNullable(type.toCatalogDataType().getPrimitiveType()); + this.nullableSpecified = nullableType == ColumnNullableType.NULLABLE + || nullableType == ColumnNullableType.NOT_NULLABLE; this.autoIncInitValue = autoIncInitValue; this.defaultValue = defaultValue; this.onUpdateDefaultValue = onUpdateDefaultValue; this.comment = comment; + this.commentSpecified = commentSpecified; this.isVisible = isVisible; this.generatedColumnDesc = generatedColumnDesc; } @@ -183,6 +207,10 @@ public boolean hasDefaultValue() { return defaultValue.isPresent(); } + public boolean hasOnUpdateDefaultValue() { + return onUpdateDefaultValue.isPresent(); + } + public boolean isVisible() { return isVisible; } @@ -203,23 +231,40 @@ public String getComment(boolean escapeQuota) { return SqlUtils.escapeQuota(comment); } + public boolean isCommentSpecified() { + return commentSpecified; + } + /** * toSql */ public String toSql() { + return toSql("`" + name + "`", true); + } + + /** + * Convert this column definition to schema-change SQL with a caller-provided column name. + * Unlike {@link #toSql()}, this overload emits COMMENT only when it was explicitly specified. + */ + public String toSql(String columnNameSql) { + return toSql(columnNameSql, commentSpecified); + } + + private String toSql(String columnNameSql, boolean includeComment) { StringBuilder sb = new StringBuilder(); - sb.append("`").append(name).append("` "); + sb.append(columnNameSql).append(" "); sb.append(type.toSql()).append(" "); if (aggType != null && aggType != AggregateType.NONE) { sb.append(aggType.name()).append(" "); } - if (!isNullable) { - sb.append("NOT NULL "); - } else { - // should append NULL to make result can be executed right. - sb.append("NULL "); + if (nullableSpecified) { + if (!isNullable) { + sb.append("NOT NULL "); + } else { + sb.append("NULL "); + } } if (autoIncInitValue != -1) { @@ -247,7 +292,9 @@ public String toSql() { sb.append("DEFAULT ").append("NULL").append(" "); } } - sb.append("COMMENT \"").append(SqlUtils.escapeQuota(comment)).append("\""); + if (includeComment) { + sb.append("COMMENT ").append(SqlLiteralUtils.quoteStringLiteral(getComment())); + } return sb.toString(); } @@ -306,10 +353,25 @@ private void checkKeyColumnType(boolean isOlap) { */ public void validate(boolean isOlap, Set keysSet, Set clusterKeySet, boolean isEnableMergeOnWrite, KeysType keysType) { + validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, false); + } + + /** + * Validate a nested field whose name is scoped by its parent path rather than the Doris top-level column namespace. + */ + public void validateNestedColumn(boolean isOlap, Set keysSet, Set clusterKeySet, + boolean isEnableMergeOnWrite, KeysType keysType) { + validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType, true); + } + + private void validateInternal(boolean isOlap, Set keysSet, Set clusterKeySet, + boolean isEnableMergeOnWrite, KeysType keysType, boolean nestedColumn) { try { // if enableAddHiddenColumn is true, can add hidden column. // So does not check if the column name starts with __DORIS_ - if (enableAddHiddenColumn) { + if (nestedColumn) { + FeNameFormat.checkColumnNameBypassSystemColumnPrefix(name); + } else if (enableAddHiddenColumn) { FeNameFormat.checkColumnNameBypassHiddenColumn(name); } else { FeNameFormat.checkColumnName(name); @@ -428,18 +490,8 @@ public void validate(boolean isOlap, Set keysSet, Set clusterKey .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) { throw new AnalysisException("Array type column default value only support null or " + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE); - } else if (type.isMapType()) { - if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Map type column default value just support null"); - } - } else if (type.isStructType()) { - if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Struct type column default value just support null"); - } - } else if (type.isJsonType() || type.isVariantType()) { - if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Json or Variant type column default value just support null"); - } + } else { + validateComplexTypeDefaultValue(); } if (!isNullable && defaultValue.isPresent() @@ -521,6 +573,22 @@ public void validate(boolean isOlap, Set keysSet, Set clusterKey validateGeneratedColumnInfo(); } + /** + * Validate non-null defaults for complex types before connector-specific validation. + */ + public void validateComplexTypeDefaultValue() throws AnalysisException { + if (!defaultValue.isPresent() || defaultValue.get() == DefaultValue.NULL_DEFAULT_VALUE) { + return; + } + if (type.isMapType()) { + throw new AnalysisException("Map type column default value just support null"); + } else if (type.isStructType()) { + throw new AnalysisException("Struct type column default value just support null"); + } else if (type.isJsonType() || type.isVariantType()) { + throw new AnalysisException("Json or Variant type column default value just support null"); + } + } + /** * translate to catalog create table stmt */ @@ -554,6 +622,8 @@ public Column translateToCatalogStyleForSchemaChange() { generatedColumnDesc.map(desc -> ConnectContextUtil.getAffectQueryResultInPlanVariables(ConnectContext.get())) .orElse(null)); + column.setNullableSpecified(nullableSpecified); + column.setCommentSpecified(commentSpecified); column.setAggregationTypeImplicit(aggTypeImplicit); return column; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java index be5e91c8a0b7d4..f9a3444cbe0394 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; @@ -42,6 +43,7 @@ */ public class DropColumnOp extends AlterTableOp { private String colName; + private ColumnPath columnPath; private String rollupName; private Map properties; @@ -50,8 +52,13 @@ public class DropColumnOp extends AlterTableOp { * DropColumnOp */ public DropColumnOp(String colName, String rollupName, Map properties) { + this(ColumnPath.of(colName), rollupName, properties); + } + + public DropColumnOp(ColumnPath columnPath, String rollupName, Map properties) { super(AlterOpType.SCHEMA_CHANGE); - this.colName = colName; + this.colName = columnPath.getLeafName(); + this.columnPath = columnPath; this.rollupName = rollupName; this.properties = properties; } @@ -60,6 +67,10 @@ public String getColName() { return colName; } + public ColumnPath getColumnPath() { + return columnPath; + } + public String getRollupName() { return rollupName; } @@ -70,7 +81,7 @@ public void validate(ConnectContext ctx) throws UserException { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, colName, FeNameFormat.getColumnNameRegex()); } - if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { + if (!columnPath.isNested() && colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { throw new AnalysisException("Do not support drop hidden column"); } @@ -171,7 +182,7 @@ public boolean allowOpRowBinlog() { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("DROP COLUMN `").append(colName).append("`"); + sb.append("DROP COLUMN ").append(columnPath.toSql()); if (rollupName != null) { sb.append(" FROM `").append(rollupName).append("`"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java index 0b1c6ac49558e2..9b0e4cadab3514 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnCommentOp.java @@ -18,8 +18,10 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; +import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; @@ -31,17 +33,25 @@ * ModifyColumnCommentOp */ public class ModifyColumnCommentOp extends AlterTableOp { - private String colName; + private ColumnPath columnPath; private String comment; public ModifyColumnCommentOp(String colName, String comment) { + this(ColumnPath.of(colName), comment); + } + + public ModifyColumnCommentOp(ColumnPath columnPath, String comment) { super(AlterOpType.MODIFY_COLUMN_COMMENT); - this.colName = colName; + this.columnPath = columnPath; this.comment = Strings.nullToEmpty(comment); } public String getColName() { - return colName; + return columnPath.getFullPath(); + } + + public ColumnPath getColumnPath() { + return columnPath; } public String getComment() { @@ -55,7 +65,7 @@ public Map getProperties() { @Override public void validate(ConnectContext ctx) throws UserException { - if (Strings.isNullOrEmpty(colName)) { + if (columnPath == null || Strings.isNullOrEmpty(columnPath.getFullPath())) { throw new AnalysisException("Empty column name"); } } @@ -79,8 +89,8 @@ public boolean allowOpRowBinlog() { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("MODIFY COLUMN COMMENT ").append(colName); - sb.append(" '").append(comment).append("'"); + sb.append("MODIFY COLUMN ").append(columnPath.toSql()); + sb.append(" COMMENT ").append(SqlLiteralUtils.quoteStringLiteral(comment)); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java index 049d12bc325ae4..f5acfbed9df9cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyColumnOp.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; @@ -43,6 +44,7 @@ */ public class ModifyColumnOp extends AlterTableOp { private ColumnDefinition columnDef; + private ColumnPath columnPath; private ColumnPosition colPos; // which rollup is to be modify, if rollup is null, modify base table. private String rollupName; @@ -54,8 +56,17 @@ public class ModifyColumnOp extends AlterTableOp { public ModifyColumnOp(ColumnDefinition columnDef, ColumnPosition colPos, String rollup, Map properties) { + this(columnDef, ColumnPath.of(columnDef.getName()), colPos, rollup, properties); + } + + /** + * Create modify-column operation with the original nested column path. + */ + public ModifyColumnOp(ColumnDefinition columnDef, ColumnPath columnPath, ColumnPosition colPos, + String rollup, Map properties) { super(AlterOpType.SCHEMA_CHANGE); this.columnDef = columnDef; + this.columnPath = columnPath; this.colPos = colPos; this.rollupName = rollup; this.properties = properties; @@ -65,6 +76,10 @@ public Column getColumn() { return column; } + public ColumnPath getColumnPath() { + return columnPath; + } + public ColumnPosition getColPos() { return colPos; } @@ -122,7 +137,11 @@ public void validate(ConnectContext ctx) throws UserException { } } } - columnDef.validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + if (columnPath.isNested()) { + columnDef.validateNestedColumn(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + } else { + columnDef.validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite, keysType); + } if (colPos != null) { colPos.analyze(); if (olapTable != null) { @@ -187,7 +206,7 @@ public boolean needChangeMTMVState() { @Override public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("MODIFY COLUMN ").append(columnDef.toSql()); + sb.append("MODIFY COLUMN ").append(columnDef.toSql(columnPath.toSql())); if (colPos != null) { sb.append(" ").append(colPos); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java index e66ae02c15337b..15270360535d85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java @@ -18,10 +18,12 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.Column; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; @@ -33,11 +35,17 @@ */ public class RenameColumnOp extends AlterTableOp { private String colName; + private ColumnPath columnPath; private String newColName; public RenameColumnOp(String colName, String newColName) { + this(ColumnPath.of(colName), newColName); + } + + public RenameColumnOp(ColumnPath columnPath, String newColName) { super(AlterOpType.RENAME); - this.colName = colName; + this.colName = columnPath.getLeafName(); + this.columnPath = columnPath; this.newColName = newColName; } @@ -45,6 +53,10 @@ public String getColName() { return colName; } + public ColumnPath getColumnPath() { + return columnPath; + } + public String getNewColName() { return newColName; } @@ -59,11 +71,15 @@ public void validate(ConnectContext ctx) throws UserException { throw new AnalysisException("New column name is not set"); } - if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { + if (!columnPath.isNested() && colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { throw new AnalysisException("Do not support rename hidden column"); } - FeNameFormat.checkColumnName(newColName); + if (columnPath.isNested()) { + FeNameFormat.checkColumnNameBypassSystemColumnPrefix(newColName); + } else { + FeNameFormat.checkColumnName(newColName); + } } @Override @@ -83,7 +99,7 @@ public boolean needChangeMTMVState() { @Override public String toSql() { - return "RENAME COLUMN " + colName + " " + newColName; + return "RENAME COLUMN " + columnPath.toSql() + " " + SqlUtils.getIdentSql(newColName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java index e584e2ec1dd42b..3568be06b0844c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java @@ -175,11 +175,6 @@ public Plan getExplainPlan(ConnectContext ctx) { private void deleteExistingFilesInFE(String tvfName, Map props) throws Exception { - String filePath = props.get("file_path"); - // Extract parent directory from prefix path: s3://bucket/path/to/prefix_ -> s3://bucket/path/to/ - String parentDir = FileSystemUtil.extractParentDirectory(filePath); - LOG.info("TVF sink: deleting existing files in directory: {}", parentDir); - // Copy props for building StorageProperties (exclude write-specific params) Map fsCopyProps = new HashMap<>(props); fsCopyProps.remove("file_path"); @@ -192,6 +187,12 @@ private void deleteExistingFilesInFE(String tvfName, Map props) fsCopyProps.remove("compress_type"); StorageProperties storageProps = StorageProperties.createPrimary(fsCopyProps); + // Concrete filesystems only accept their native schemes; normalize legacy compatibility + // schemes (e.g. cos:// with s3.* properties) before crossing the plugin boundary. + String filePath = storageProps.validateAndNormalizeUri(props.get("file_path")); + // Extract parent directory from prefix path: s3://bucket/path/to/prefix_ -> s3://bucket/path/to/ + String parentDir = FileSystemUtil.extractParentDirectory(filePath); + LOG.info("TVF sink: deleting existing files in directory: {}", parentDir); try (org.apache.doris.filesystem.FileSystem fs = FileSystemFactory.getFileSystem(storageProps)) { fs.delete(Location.of(parentDir), true); } catch (java.io.IOException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java index 390b66076d4859..d4d65dfd225302 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java @@ -24,7 +24,6 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.common.Pair; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.trees.plans.algebra.Intersect; import org.apache.doris.nereids.trees.plans.distribute.DistributeContext; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker; import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager; @@ -32,12 +31,14 @@ import org.apache.doris.nereids.util.Utils; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.HashJoinNode; +import org.apache.doris.planner.IntersectNode; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.ScanNode; import org.apache.doris.planner.SetOperationNode; import org.apache.doris.qe.ConnectContext; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; @@ -352,7 +353,8 @@ private void collectScanRanges( } } - private boolean shouldFillUpInstances(List hashJoinNodes, List setOperationNodes) { + @VisibleForTesting + static boolean shouldFillUpInstances(List hashJoinNodes, List setOperationNodes) { for (HashJoinNode hashJoinNode : hashJoinNodes) { if (!hashJoinNode.isBucketShuffle()) { continue; @@ -368,7 +370,11 @@ private boolean shouldFillUpInstances(List hashJoinNodes, List computeAsteriskOutput() { @Override public boolean supportPruneNestedColumn() { ExternalTable table = getTable(); - if (table instanceof IcebergExternalTable || table instanceof IcebergSysExternalTable) { + if (table instanceof IcebergExternalTable) { return true; + } else if (table instanceof IcebergSysExternalTable) { + // Position deletes use the native reader, which supports nested column pruning. Other + // Iceberg system tables are materialized as StructLike rows by the SDK and consumed by + // ordinal in the JNI reader, so their nested struct layout must remain unchanged. + return ((IcebergSysExternalTable) table).isPositionDeletesTable(); } else if (table instanceof HMSExternalTable) { HMSExternalTable hmsTable = (HMSExternalTable) table; if (hmsTable.getDlaType() == HMSExternalTable.DLAType.HUDI) { @@ -268,7 +274,8 @@ public static class SelectedPartitions { // NOT_PRUNED means the Nereids planner does not handle the partition pruning. // This can be treated as the initial value of SelectedPartitions. // Or used to indicate that the partition pruning is not processed. - public static SelectedPartitions NOT_PRUNED = new SelectedPartitions(0, ImmutableMap.of(), false, false); + public static SelectedPartitions NOT_PRUNED = new SelectedPartitions(0, ImmutableMap.of(), false, false, + Optional.empty()); /** * total partition number */ @@ -288,12 +295,19 @@ public static class SelectedPartitions { */ public final boolean hasPartitionPredicate; + /** + * sorted partition ranges for binary search filtering. + * Frozen at construction time to ensure consistency with selectedPartitions. + * Empty if binary search is not applicable (e.g., default partition only). + */ + public final Optional> sortedPartitionRanges; + /** * Constructor for SelectedPartitions. */ public SelectedPartitions(long totalPartitionNum, Map selectedPartitions, boolean isPruned) { - this(totalPartitionNum, selectedPartitions, isPruned, false); + this(totalPartitionNum, selectedPartitions, isPruned, false, Optional.empty()); } /** @@ -301,11 +315,21 @@ public SelectedPartitions(long totalPartitionNum, Map sel */ public SelectedPartitions(long totalPartitionNum, Map selectedPartitions, boolean isPruned, boolean hasPartitionPredicate) { + this(totalPartitionNum, selectedPartitions, isPruned, hasPartitionPredicate, Optional.empty()); + } + + /** + * Constructor for SelectedPartitions with sorted partition ranges. + */ + public SelectedPartitions(long totalPartitionNum, Map selectedPartitions, + boolean isPruned, boolean hasPartitionPredicate, + Optional> sortedPartitionRanges) { this.totalPartitionNum = totalPartitionNum; this.selectedPartitions = ImmutableMap.copyOf(Objects.requireNonNull(selectedPartitions, "selectedPartitions is null")); this.isPruned = isPruned; this.hasPartitionPredicate = hasPartitionPredicate; + this.sortedPartitionRanges = sortedPartitionRanges; } @Override @@ -320,12 +344,14 @@ public boolean equals(Object o) { return isPruned == that.isPruned && hasPartitionPredicate == that.hasPartitionPredicate && Objects.equals( - selectedPartitions.keySet(), that.selectedPartitions.keySet()); + selectedPartitions.keySet(), that.selectedPartitions.keySet()) + && Objects.equals( + sortedPartitionRanges.isPresent(), that.sortedPartitionRanges.isPresent()); } @Override public int hashCode() { - return Objects.hash(selectedPartitions, isPruned, hasPartitionPredicate); + return Objects.hash(selectedPartitions, isPruned, hasPartitionPredicate, sortedPartitionRanges.isPresent()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableStreamScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableStreamScan.java index dd7c06792c63e4..73a4d0441b4ca8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableStreamScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableStreamScan.java @@ -276,11 +276,11 @@ public LogicalOlapTableStreamScan withSelectedPartitionIds(List selectedPa */ @Override public LogicalOlapTableStreamScan withSelectedPartitionIds(List selectedPartitionIds, - boolean isPartitionPruned) { + boolean hasPartitionPredicate) { return AbstractPlan.copyWithSameId(this, () -> new LogicalOlapTableStreamScan(relationId, (Table) table, qualifier, groupExpression, Optional.of(getLogicalProperties()), - selectedPartitionIds, isPartitionPruned, hasPartitionPredicate, selectedTabletIds, + selectedPartitionIds, true, hasPartitionPredicate, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, cachedOutput, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots, virtualColumns, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java index e3be64e577c617..2eb9bc0a07d030 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java @@ -180,7 +180,7 @@ public PhysicalLazyMaterialize(CHILD_TYPE child, } outputBuilder.add(outputSlot); lazyColumnForRel.add(originalColumn); - lazyBaseColumnIdxForRel.add(relationTable.getBaseColumnIdxByName(lazySlot.getName())); + lazyBaseColumnIdxForRel.add(relationTable.getBaseColumnIdxByName(originalColumn.getName())); lazySlotLocationForRel.add(loc); loc++; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java index a0af54c7ed09c5..1f5c843e292428 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java @@ -44,7 +44,6 @@ * Physical SetOperation. */ public abstract class PhysicalSetOperation extends AbstractPhysicalPlan implements SetOperation { - public static final String DISTRIBUTE_TO_CHILD_INDEX = "DistributeToChildIndex"; protected final Qualifier qualifier; protected final List outputs; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java index 2db9c1afc27492..1d4255b6df98ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; @@ -43,21 +44,30 @@ public class PhysicalStorageLayerAggregate extends PhysicalCatalogRelation { private final PhysicalCatalogRelation relation; private final PushDownAggOp aggOp; + private final List countArgumentExprIds; public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp) { + this(relation, aggOp, ImmutableList.of()); + } + + public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp, + List countArgumentExprIds) { super(relation.getRelationId(), relation.getType(), relation.getTable(), relation.getQualifier(), Optional.empty(), relation.getLogicalProperties(), ImmutableList.of()); this.relation = Objects.requireNonNull(relation, "relation cannot be null"); this.aggOp = Objects.requireNonNull(aggOp, "aggOp cannot be null"); + this.countArgumentExprIds = ImmutableList.copyOf(countArgumentExprIds); } public PhysicalStorageLayerAggregate(PhysicalCatalogRelation relation, PushDownAggOp aggOp, + List countArgumentExprIds, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(relation.getRelationId(), relation.getType(), relation.getTable(), relation.getQualifier(), groupExpression, logicalProperties, physicalProperties, statistics, ImmutableList.of()); this.relation = Objects.requireNonNull(relation, "relation cannot be null"); this.aggOp = Objects.requireNonNull(aggOp, "aggOp cannot be null"); + this.countArgumentExprIds = ImmutableList.copyOf(countArgumentExprIds); } public PhysicalRelation getRelation() { @@ -68,6 +78,10 @@ public PushDownAggOp getAggOp() { return aggOp; } + public List getCountArgumentExprIds() { + return countArgumentExprIds; + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitPhysicalStorageLayerAggregate(this, context); @@ -83,20 +97,21 @@ public String toString() { } public PhysicalStorageLayerAggregate withPhysicalOlapScan(PhysicalOlapScan physicalOlapScan) { - return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate(physicalOlapScan, aggOp)); + return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate( + physicalOlapScan, aggOp, countArgumentExprIds)); } @Override public PhysicalStorageLayerAggregate withGroupExpression(Optional groupExpression) { return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate(relation, aggOp, - groupExpression, getLogicalProperties(), physicalProperties, statistics)); + countArgumentExprIds, groupExpression, getLogicalProperties(), physicalProperties, statistics)); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate(relation, aggOp, - groupExpression, logicalProperties.get(), physicalProperties, statistics)); + countArgumentExprIds, groupExpression, logicalProperties.get(), physicalProperties, statistics)); } @Override @@ -104,7 +119,7 @@ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalPr Statistics statistics) { return AbstractPlan.copyWithSameId(this, () -> new PhysicalStorageLayerAggregate( (PhysicalCatalogRelation) relation.withPhysicalPropertiesAndStats(null, statistics), - aggOp, groupExpression, + aggOp, countArgumentExprIds, groupExpression, getLogicalProperties(), physicalProperties, statistics)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index a47352a29e60cc..c6a5dc1b0285ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -468,7 +468,8 @@ public static DataType fromCatalogType(Type type) { if (type.isStructType()) { List structFields = ((org.apache.doris.catalog.StructType) (type)).getFields().stream() .map(cf -> new StructField(cf.getName(), fromCatalogType(cf.getType()), - cf.getContainsNull(), cf.getComment() == null ? "" : cf.getComment())) + cf.getContainsNull(), cf.getComment() == null ? "" : cf.getComment(), + cf.isCommentSpecified())) .collect(ImmutableList.toImmutableList()); return new StructType(structFields); } else if (type.isMapType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java index 63e94bc369bb0c..aefcf20f227ff1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java @@ -17,6 +17,9 @@ package org.apache.doris.nereids.types; +import org.apache.doris.common.util.SqlUtils; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.util.SqlLiteralUtils; import org.apache.doris.nereids.util.Utils; import java.util.Objects; @@ -32,6 +35,7 @@ public class StructField { private final DataType dataType; private final boolean nullable; private final String comment; + private final boolean commentSpecified; /** * StructField Constructor @@ -40,10 +44,16 @@ public class StructField { * @param nullable Indicates if values of this field can be `null` values */ public StructField(String name, DataType dataType, boolean nullable, String comment) { + this(name, dataType, nullable, comment, comment != null && !comment.isEmpty()); + } + + public StructField(String name, DataType dataType, boolean nullable, String comment, + boolean commentSpecified) { this.name = Objects.requireNonNull(name, "name should not be null").toLowerCase(); this.dataType = Objects.requireNonNull(dataType, "dataType should not be null"); this.nullable = nullable; this.comment = Objects.requireNonNull(comment, "comment should not be null"); + this.commentSpecified = commentSpecified; } public String getName() { @@ -62,6 +72,10 @@ public String getComment() { return comment; } + public boolean isCommentSpecified() { + return commentSpecified; + } + public StructField conversion() { if (this.dataType.equals(dataType.conversion())) { return this; @@ -70,22 +84,24 @@ public StructField conversion() { } public StructField withDataType(DataType dataType) { - return new StructField(name, dataType, nullable, comment); + return new StructField(name, dataType, nullable, comment, commentSpecified); } public StructField withDataTypeAndNullable(DataType dataType, boolean nullable) { - return new StructField(name, dataType, nullable, comment); + return new StructField(name, dataType, nullable, comment, commentSpecified); } public org.apache.doris.catalog.StructField toCatalogDataType() { return new org.apache.doris.catalog.StructField( - name, dataType.toCatalogDataType(), comment, nullable); + name, dataType.toCatalogDataType(), comment, nullable, commentSpecified); } public String toSql() { - return name + ":" + dataType.toSql() + String nameSql = NereidsParser.isValidUnquotedIdentifier(name) ? name : SqlUtils.getIdentSql(name); + return nameSql + ":" + dataType.toSql() + (nullable ? "" : " NOT NULL") - + (comment.isEmpty() ? "" : " COMMENT " + comment); + + (!commentSpecified ? "" : " COMMENT " + + SqlLiteralUtils.quoteStringLiteral(comment)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java index 947da096185821..39ced0eddec48f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java @@ -40,6 +40,7 @@ import com.google.common.collect.Lists; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -62,10 +63,11 @@ public static AggregateFunction tryConvertToMultiDistinct(AggregateFunction func /**countDistinctMultiExprToCountIf*/ public static Expression countDistinctMultiExprToCountIf(Count count) { - Set arguments = ImmutableSet.copyOf(count.getArguments()); - Expression countExpr = count.getArgument(arguments.size() - 1); - for (int i = arguments.size() - 2; i >= 0; --i) { - Expression argument = count.getArgument(i); + Iterator arguments = ImmutableSet.copyOf(count.getArguments()) + .asList().reverse().iterator(); + Expression countExpr = arguments.next(); + while (arguments.hasNext()) { + Expression argument = arguments.next(); If ifNull = new If(new IsNull(argument), NullLiteral.INSTANCE, countExpr); countExpr = assignNullType(ifNull); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java new file mode 100644 index 00000000000000..5d87bc08f5f56d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/SqlLiteralUtils.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import org.apache.doris.qe.SqlModeHelper; + +/** + * Utilities for decoding and rendering SQL string literals. + */ +public final class SqlLiteralUtils { + + private SqlLiteralUtils() { + } + + /** + * Decode backslash escape sequences used in SQL string literals. + */ + public static String unescapeBackSlash(String value) { + StringBuilder result = new StringBuilder(); + int length = value.length(); + for (int i = 0; i < length; ++i) { + char current = value.charAt(i); + if (current == '\\' && (i + 1) < length) { + switch (value.charAt(i + 1)) { + case 'n': + result.append('\n'); + break; + case 't': + result.append('\t'); + break; + case 'r': + result.append('\r'); + break; + case 'b': + result.append('\b'); + break; + case '0': + result.append('\0'); + break; + case 'Z': + result.append('\032'); + break; + case '_': + case '%': + result.append('\\'); + result.append(value.charAt(i + 1)); + break; + default: + result.append(value.charAt(i + 1)); + break; + } + i++; + } else { + result.append(current); + } + } + return result.toString(); + } + + /** + * Decode a STRING_LITERAL token according to the current SQL mode. + */ + public static String parseStringLiteral(String text) { + String value = text.substring(1, text.length() - 1); + if (text.charAt(0) == '\'') { + value = value.replace("''", "'"); + } else { + value = value.replace("\"\"", "\""); + } + return SqlModeHelper.hasNoBackSlashEscapes() ? value : unescapeBackSlash(value); + } + + /** + * Quote a value as a STRING_LITERAL that can be parsed under the current SQL mode. + */ + public static String quoteStringLiteral(String value) { + String escaped = SqlModeHelper.hasNoBackSlashEscapes() + ? value : value.replace("\\", "\\\\"); + return "\"" + escaped.replace("\"", "\"\"") + "\""; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java index e5820c4e426755..786e026660af39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java @@ -89,6 +89,10 @@ public void unsetNeedsFinalize() { updateplanNodeName(); } + public boolean isNeedsFinalize() { + return needsFinalize; + } + // Used by new optimizer public void setUseStreamingPreagg(boolean useStreamingPreagg) { this.useStreamingPreagg = useStreamingPreagg; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java index 4af4ba17e18578..9775f81c172fae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergMergeSink.java @@ -131,6 +131,7 @@ public void bindDataSink(Optional insertCtx) } tSink.setFormatVersion(formatVersion); tSink.setSchemaJson(SchemaParser.toJson(schema)); + tSink.setCollectColumnStats(IcebergUtils.shouldCollectColumnStats(icebergTable, schema)); // partition spec if (icebergTable.spec().isPartitioned()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java index 0f3b1bb24d26bc..b7d3da47cb4d45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java @@ -135,6 +135,7 @@ public void bindDataSink(Optional insertCtx) schema = IcebergUtils.appendRowLineageFieldsForV3(schema); } tSink.setSchemaJson(SchemaParser.toJson(schema)); + tSink.setCollectColumnStats(IcebergUtils.shouldCollectColumnStats(icebergTable, schema)); // partition spec if (icebergTable.spec().isPartitioned()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java index f939102d699a13..66eda40079952f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/LocalExchangeNode.java @@ -283,8 +283,15 @@ public LocalExchangeType preferType() { @Override public LocalExchangeTypeRequire autoRequireHash() { - if (requireType == LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE - || requireType == LocalExchangeType.BUCKET_HASH_SHUFFLE) { + // Callers are pass-through operators (union / streaming agg / sort) that report + // resolveExchangeType(requireChild) upward while leaving row placement to their + // children. A specific hash require must therefore be forwarded as-is: degrading + // LOCAL_EXECUTION_HASH_SHUFFLE to the generic RequireHash lets a bucket-distributed + // child satisfy the requirement and keep its bucket placement, while the operator + // still claims LOCAL_EXECUTION_HASH_SHUFFLE to its parent — the parent (e.g. a + // bucket join upgraded to local hash) then skips its re-align local exchange and + // the mixed placements compute wrong results. + if (requireType.isHashShuffle()) { return this; } return RequireHash.INSTANCE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index c1f97fdeb96e20..1fc55fe8f5e1c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -771,11 +771,18 @@ public void setCardinalityAfterFilter(long cardinalityAfterFilter) { } protected TPushAggOp pushDownAggNoGroupingOp = TPushAggOp.NONE; + // Explicit COUNT arguments. COUNT(*)/COUNT(1) intentionally keep this empty even though + // column pruning retains one placeholder scan slot. + protected List pushDownCountSlotIds = Collections.emptyList(); public void setPushDownAggNoGrouping(TPushAggOp pushDownAggNoGroupingOp) { this.pushDownAggNoGroupingOp = pushDownAggNoGroupingOp; } + public void setPushDownCountSlotIds(List pushDownCountSlotIds) { + this.pushDownCountSlotIds = Lists.newArrayList(pushDownCountSlotIds); + } + public void setChildrenDistributeExprLists(List> childrenDistributeExprLists) { this.childrenDistributeExprLists = childrenDistributeExprLists; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java index 4074ab3e85ae6e..1396701d3c72dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java @@ -223,7 +223,15 @@ public Pair enforceAndDeriveLocalExchange(PlanTrans : LocalExchangeType.NOOP; } else { // Intersect / Except - if (AddLocalExchange.isColocated(this)) { + if (AddLocalExchange.isColocated(this) || isBucketShuffle()) { + // COLOCATE / BUCKET_SHUFFLE: every child is distributed by the basic child's + // storage bucket function (basic side scans buckets directly, other sides come + // from bucket-shuffle exchanges), so all children must stay aligned by that + // bucket function locally. requireBucketHash keeps bucket-distributed children + // as-is and re-aligns a serial (NOOP-claim) child with a BUCKET_HASH_SHUFFLE + // local exchange — same pattern as HashJoinNode's colocate/bucket-shuffle + // branch. An execution-hash require here would locally re-partition one side + // by a different hash function and break build/probe alignment. requireChild = LocalExchangeTypeRequire.requireBucketHash(); outputType = LocalExchangeType.BUCKET_HASH_SHUFFLE; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java index b74ebd0e7c3aca..5212c5a5fd6533 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java @@ -21,6 +21,7 @@ package org.apache.doris.planner.normalize; import org.apache.doris.analysis.DescriptorTable; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Tablet; import org.apache.doris.common.Pair; @@ -98,6 +99,7 @@ private Optional setQueryCacheParam( queryCacheParam.setForceRefreshQueryCache(sessionVariable.isQueryCacheForceRefresh()); queryCacheParam.setEntryMaxBytes(sessionVariable.getQueryCacheEntryMaxBytes()); queryCacheParam.setEntryMaxRows(sessionVariable.getQueryCacheEntryMaxRows()); + queryCacheParam.setAllowIncremental(computeAllowIncremental(cachePoint, sessionVariable)); queryCacheParam.setOutputSlotMapping( cachePoint.cacheRoot.getOutputTupleIds() @@ -145,6 +147,74 @@ private Optional doComputeCachePoint(PlanNode planRoot) { return Optional.empty(); } + /** + * Decide whether BE may serve a stale cache entry by scanning only the delta + * rowsets since the cached version and emitting them together with the cached + * partial aggregation blocks (the upstream aggregation merges both). + * + *

This is only correct when all of the following hold: + *

    + *
  • The cache point aggregation does not finalize: its output is a partial + * state that an upstream aggregation always merges, so emitting the cached + * blocks and the delta blocks side by side yields the correct result. A + * finalized output has no downstream merge, and the two emissions would + * produce duplicated group keys.
    • + *
  • The cache point aggregates the raw detail rows directly (its child is + * the olap scan node): with another aggregation in between, that inner + * aggregation would see only the delta rows during an incremental run, and + * its finalized output over the delta is not a mergeable complement of its + * output over the cached snapshot.
    • + *
  • The scanned index is append-only, guaranteeing "cached snapshot + + * delta rowsets == new snapshot": either DUP_KEYS, or merge-on-write + * UNIQUE_KEYS, for which BE additionally verifies per tablet (through the + * delete bitmap of the delta window) that no pre-existing key was rewritten + * and falls back otherwise. Merge-on-read UNIQUE resolves duplicates by + * merging across rowsets at read time, so a delta-only scan cannot stand + * alone there; AGG tables merge rows inside the storage layer likewise. + * DELETE predicates are handled on BE: a delta containing delete predicates + * falls back to a full recompute.
    • + *
+ */ + private boolean computeAllowIncremental(CachePoint cachePoint, SessionVariable sessionVariable) { + if (!sessionVariable.getEnableQueryCacheIncremental()) { + return false; + } + // The cache point is always an aggregation node (see doComputeCachePoint). + if (((AggregationNode) cachePoint.cacheRoot).isNeedsFinalize()) { + return false; + } + // The cached partial state and the delta partial state merge correctly + // only when the cache point aggregates the raw detail rows directly. + // With a nested cache point (partial agg over a finalized/deduplicating + // agg over scan), the inner agg sees only the delta rows during an + // incremental run, so its finalized output over the delta is NOT a + // mergeable complement of its output over the cached snapshot (e.g. + // "group by cnt" buckets computed from partial counts are simply wrong). + if (!(cachePoint.cacheRoot.getChild(0) instanceof OlapScanNode)) { + return false; + } + OlapScanNode scanNode = (OlapScanNode) cachePoint.cacheRoot.getChild(0); + OlapTable olapTable = scanNode.getOlapTable(); + long selectIndexId = scanNode.getSelectedIndexId() == -1 + ? olapTable.getBaseIndexId() + : scanNode.getSelectedIndexId(); + // Note: judged on the selected index, not the base table: a DUP table + // may serve the query from an aggregated materialized view, whose data + // is no longer append-only. + KeysType keysType = olapTable.getKeysTypeByIndexId(selectIndexId); + if (keysType == KeysType.DUP_KEYS) { + return true; + } + // A merge-on-write UNIQUE index is append-only as long as a load does + // not touch pre-existing keys, which covers the common "hourly append + // plus occasional backfill" pattern: BE verifies per tablet through + // the delete bitmap of the delta window and falls back to a full + // recompute for the rare load that rewrites history. A merge-on-read + // UNIQUE index resolves duplicates by merging across rowsets at read + // time, so a delta-only scan cannot stand alone there. + return keysType == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite(); + } + private List normalizePlanTree(ConnectContext context, CachePoint cachePoint) { List normalizedPlans = new ArrayList<>(); doNormalizePlanTree(context, cachePoint.digestRoot, normalizedPlans); diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java index fb9ffde02dc3c9..73a78e7356ed66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java @@ -21,8 +21,11 @@ import org.apache.doris.catalog.InternalSchema; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.util.HttpURLUtil; +import org.apache.doris.common.util.InternalHttpsUtils; import org.apache.doris.qe.GlobalVariable; +import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -34,23 +37,23 @@ import java.net.URL; import java.util.Calendar; import java.util.stream.Collectors; +import javax.net.ssl.HttpsURLConnection; public class AuditStreamLoader { private static final Logger LOG = LogManager.getLogger(AuditStreamLoader.class); - private static String loadUrlPattern = "http://%s/api/%s/%s/_stream_load?"; // timeout for both connection and read. 10 seconds is long enough. private static final int HTTP_TIMEOUT_MS = 10000; - private String hostPort; private String db; private String auditLogTbl; private String auditLogLoadUrlStr; private String feIdentity; public AuditStreamLoader() { - this.hostPort = "127.0.0.1:" + Config.http_port; this.db = FeConstants.INTERNAL_DB_NAME; this.auditLogTbl = AuditLoader.AUDIT_LOG_TABLE; - this.auditLogLoadUrlStr = String.format(loadUrlPattern, hostPort, db, auditLogTbl); + String scheme = Config.enable_https ? "https" : "http"; + String hostPort = "127.0.0.1:" + HttpURLUtil.getHttpPort(); + this.auditLogLoadUrlStr = scheme + "://" + hostPort + "/api/" + db + "/" + auditLogTbl + "/_stream_load?"; // currently, FE identity is FE's IP:port, so we replace the "." and ":" to make it suitable for label this.feIdentity = Env.getCurrentEnv().getSelfNode().getIdent().replaceAll("\\.", "_").replaceAll(":", "_"); } @@ -58,6 +61,11 @@ public AuditStreamLoader() { private HttpURLConnection getConnection(String urlStr, String label, String clusterToken) throws IOException { URL url = new URL(urlStr); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + if (conn instanceof HttpsURLConnection && Config.enable_https) { + HttpsURLConnection httpsConn = (HttpsURLConnection) conn; + httpsConn.setSSLSocketFactory(InternalHttpsUtils.getSslContext().getSocketFactory()); + httpsConn.setHostnameVerifier(NoopHostnameVerifier.INSTANCE); + } conn.setInstanceFollowRedirects(false); conn.setRequestMethod("PUT"); conn.setRequestProperty("token", clusterToken); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index cc24f38f4d3f27..c79b8d824ddade 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -934,6 +934,40 @@ public void clear() { statementContext = null; } + // Arrow Flight SQL only. + // Executors of already-planned queries whose results are produced on the BE and pulled later + // during the DoGet phase. Their coordinators must stay alive until the BE finishes scanning: + // an external-table scan in batch mode lazily fetches splits from the FE (a batch SplitSource + // held by the coordinator's scan nodes), so closing the coordinator at the end of + // GetFlightInfo would release the SplitSource too early and make the BE's fetchSplitBatch fail + // with "Split source X is released". These executors are finalized when the next query starts + // on this connection, or when the connection is torn down. See #62259. + private final List flightSqlDeferredExecutors = new ArrayList<>(); + + public void addFlightSqlDeferredExecutor(StmtExecutor executor) { + synchronized (flightSqlDeferredExecutors) { + flightSqlDeferredExecutors.add(executor); + } + } + + public void closeFlightSqlDeferredExecutors() { + List toClose; + synchronized (flightSqlDeferredExecutors) { + if (flightSqlDeferredExecutors.isEmpty()) { + return; + } + toClose = new ArrayList<>(flightSqlDeferredExecutors); + flightSqlDeferredExecutors.clear(); + } + for (StmtExecutor deferredExecutor : toClose) { + try { + deferredExecutor.finalizeArrowFlightQuery(); + } catch (Throwable t) { + LOG.warn("failed to finalize deferred arrow flight executor", t); + } + } + } + /** * This method is idempotent. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java index 067db673049c79..892dd3cfd732f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java @@ -75,15 +75,15 @@ public void syncJournal() throws Exception { waitOnReplaying(); } + // The master handles the group commit shortcuts without writing a journal, so the result carries + // no journal id. Waiting on journal 0 is a no-op that only logs one line per request. public long getGroupCommitLoadBeId(long tableId, String cluster) throws Exception { result = forward(buildGetGroupCommitLoadBeIdParmas(tableId, cluster)); - waitOnReplaying(); return result.groupCommitLoadBeId; } public void updateLoadData(long tableId, long receiveData) throws Exception { result = forward(buildUpdateLoadDataParams(tableId, receiveData)); - waitOnReplaying(); } private TMasterOpRequest buildSyncJournalParams() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 3c3541280755de..df3fcbac6e0ec4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -202,6 +202,7 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_SQL_CACHE = "enable_sql_cache"; public static final String ENABLE_HIVE_SQL_CACHE = "enable_hive_sql_cache"; public static final String ENABLE_QUERY_CACHE = "enable_query_cache"; + public static final String ENABLE_QUERY_CACHE_INCREMENTAL = "enable_query_cache_incremental"; public static final String QUERY_CACHE_FORCE_REFRESH = "query_cache_force_refresh"; public static final String QUERY_CACHE_ENTRY_MAX_BYTES = "query_cache_entry_max_bytes"; public static final String QUERY_CACHE_ENTRY_MAX_ROWS = "query_cache_entry_max_rows"; @@ -1166,7 +1167,7 @@ public static double getHotValueThreshold() { "FileScanNode 扫描数据的最大并发，默认为 16", "The max threads to read data of FileScanNode, default 16"}) public int maxFileScannersConcurrency = 16; - @VarAttrDef.VarAttr(name = ENABLE_FILE_SCANNER_V2, needForward = true, description = { + @VarAttrDef.VarAttr(name = ENABLE_FILE_SCANNER_V2, needForward = true, fuzzy = true, description = { "开启后 FileScanNode 会在支持的查询场景使用 FileScannerV2，默认开启", "When enabled, FileScanNode uses FileScannerV2 for supported query scans. Enabled by default."}) public boolean enableFileScannerV2 = true; @@ -1544,16 +1545,57 @@ public enum IgnoreSplitType { @VarAttrDef.VarAttr(name = ENABLE_HIVE_SQL_CACHE, fuzzy = false) public boolean enableHiveSqlCache = false; - @VarAttrDef.VarAttr(name = ENABLE_QUERY_CACHE, fuzzy = true) + // Forwarded because query cache normalization runs wherever the statement is + // planned: a forwarded statement is planned by the master in a fresh + // ConnectContext, which starts from the master's global value and then sees + // only what getForwardVariables() sends, so without this a session-level + // setting (or a SET_VAR hint) never reaches the planner: the cache follows + // the master's global instead, and enable_query_cache_incremental, which + // requires this switch, would arrive alone and never take effect. + @VarAttrDef.VarAttr(name = ENABLE_QUERY_CACHE, fuzzy = true, needForward = true) public boolean enableQueryCache = false; - @VarAttrDef.VarAttr(name = QUERY_CACHE_FORCE_REFRESH) + // Allow BE to reuse a stale query cache entry by scanning only the delta + // rowsets since the cached version and merging them with the cached partial + // aggregation blocks. Only takes effect when the cache point aggregation is + // non-finalize (its output is merged again upstream) and the selected index + // is append-only: DUP_KEYS, or merge-on-write UNIQUE_KEYS whose delta did + // not rewrite pre-existing keys (BE checks the delete bitmap per tablet); + // BE falls back to a full recompute whenever the delta cannot be captured + // (compacted away), contains delete predicates or rewrites history rows. + // Experimental: shown as `experimental_enable_query_cache_incremental` and + // settable with or without the prefix; to be promoted to EXPERIMENTAL_ONLINE + // once it graduates. + @VarAttrDef.VarAttr(name = ENABLE_QUERY_CACHE_INCREMENTAL, + varType = VariableAnnotation.EXPERIMENTAL, needForward = true, + description = {"是否允许 BE 以增量合并的方式复用过期的 Query Cache 条目：只扫描缓存版本之后的" + + "增量 rowset，并与缓存的聚合中间结果一起交给上游合并。仅对聚合直压扫描且不做 finalize " + + "的缓存点生效，且选中索引须为追加写：DUP_KEYS 表，或增量窗口内未改写既有主键的写时合并" + + "（merge-on-write）UNIQUE_KEYS 表（BE 按 tablet 检查 delete bitmap）；增量不可捕获（如" + + "已被 compaction 合并）、含 DELETE 谓词或改写了历史行时自动回退全量重算。需与 " + + "enable_query_cache 同时开启。", + "Whether BE may reuse a stale query cache entry by incremental merge: scan only" + + " the delta rowsets since the cached version and emit them together with the" + + " cached partial aggregation blocks for the upstream merge. Only takes effect" + + " when the cache point is a non-finalize aggregation directly over the scan" + + " and the selected index is append-only: DUP_KEYS, or merge-on-write" + + " UNIQUE_KEYS whose delta did not rewrite pre-existing keys (BE checks the" + + " delete bitmap per tablet); falls back to a full recompute whenever the" + + " delta cannot be captured (e.g. compacted away), contains delete predicates" + + " or rewrites history rows. Requires enable_query_cache."}) + public boolean enableQueryCacheIncremental = false; + + // Forwarded for the same reason as enable_query_cache: the master builds + // the query cache param when it plans a forwarded statement, so without + // this a forwarded statement would silently ignore a forced refresh and + // size the entries by the master's defaults instead of the session's. + @VarAttrDef.VarAttr(name = QUERY_CACHE_FORCE_REFRESH, needForward = true) private boolean queryCacheForceRefresh = false; - @VarAttrDef.VarAttr(name = QUERY_CACHE_ENTRY_MAX_BYTES) + @VarAttrDef.VarAttr(name = QUERY_CACHE_ENTRY_MAX_BYTES, needForward = true) private long queryCacheEntryMaxBytes = 5242880; - @VarAttrDef.VarAttr(name = QUERY_CACHE_ENTRY_MAX_ROWS) + @VarAttrDef.VarAttr(name = QUERY_CACHE_ENTRY_MAX_ROWS, needForward = true) private long queryCacheEntryMaxRows = 500000; @VarAttrDef.VarAttr(name = ENABLE_CONDITION_CACHE) @@ -3841,6 +3883,10 @@ public void initFuzzyModeVariables() { this.enableLocalExchange = random.nextBoolean(); this.enableSharedExchangeSinkBuffer = random.nextBoolean(); this.useSerialExchange = random.nextBoolean(); + // Randomize the external file scanner engine (FileScannerV2 vs the legacy V1 path). Kept + // here rather than in setFuzzyForCatalog() so it also runs in the external regression + // pipeline, which enables fuzzy sessions with fuzzy_test_type=p1 (not "external"). + this.enableFileScannerV2 = random.nextBoolean(); this.disableStreamPreaggregations = random.nextBoolean(); this.enableStreamingAggHashJoinForcePassthrough = random.nextBoolean(); this.enableLocalExchangeBeforeAgg = random.nextBoolean(); @@ -4676,6 +4722,14 @@ public void setEnableQueryCache(boolean enableQueryCache) { this.enableQueryCache = enableQueryCache; } + public boolean getEnableQueryCacheIncremental() { + return enableQueryCacheIncremental; + } + + public void setEnableQueryCacheIncremental(boolean enableQueryCacheIncremental) { + this.enableQueryCacheIncremental = enableQueryCacheIncremental; + } + public boolean isQueryCacheForceRefresh() { return queryCacheForceRefresh; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index dac8b50ac05297..795a81c8a76ed7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -199,6 +199,10 @@ public class StmtExecutor { @Setter private volatile Coordinator coord = null; + // Arrow Flight SQL: when true, this query's coordinator is kept alive past GetFlightInfo and + // is finalized later by ConnectContext (see #62259), so the eager close in executeAndSendResult + // is skipped. + private volatile boolean deferredForArrowFlight = false; private MasterOpExecutor masterOpExecutor = null; private RedirectStatus redirectStatus = null; private Planner planner; @@ -1035,6 +1039,23 @@ public void finalizeQuery() { QeProcessorImpl.INSTANCE.unregisterQuery(context.queryId()); } + public boolean isDeferredForArrowFlight() { + return deferredForArrowFlight; + } + + // Finalize an Arrow Flight query whose coordinator was kept alive across the + // GetFlightInfo -> DoGet phases: close the coordinator (releasing external-table batch + // SplitSources and the query queue slot) and then unregister the query. See #62259. + public void finalizeArrowFlightQuery() { + try { + if (coord != null) { + coord.close(); + } + } finally { + finalizeQuery(); + } + } + private void handleQueryWithRetry(TUniqueId queryId) throws Exception { // queue query here int retryTime = Config.max_query_retry_time; @@ -1470,6 +1491,22 @@ public void executeAndSendResult(boolean isOutfileQuery, boolean isSendFields, if (context.getConnectType().equals(ConnectType.ARROW_FLIGHT_SQL)) { Preconditions.checkState(!context.isReturnResultFromLocal()); profile.getSummaryProfile().setTempStartTime(); + // Defer closing the coordinator to ConnectContext (closed on the next query or + // connection teardown) instead of in the finally block below. This gate covers + // every Arrow Flight query whose results are produced on the BE (coordBase == + // coord) -- internal-table and external, batch or not. It is REQUIRED only for an + // external-table scan in batch mode, where the BE lazily fetches splits from the FE + // during the later DoGet phase, so closing the coordinator here would release its + // batch SplitSource too early and break DoGet. Other remote-result queries do not + // need deferral (the BE buffers their result independently) but are captured by the + // same gate; the trade-off is their coordinator, query queue slot and query + // registration stay held until the next query / teardown instead of being released + // at the end of GetFlightInfo. Point queries use a different coordBase (not + // deferred). See #62259. + if (coordBase == coord) { + deferredForArrowFlight = true; + context.addFlightSqlDeferredExecutor(this); + } return; } @@ -1579,7 +1616,12 @@ public void executeAndSendResult(boolean isOutfileQuery, boolean isSendFields, this.coord = null; throw e; } finally { - coordBase.close(); + // For deferred Arrow Flight queries the coordinator is closed later by ConnectContext + // (next query / connection teardown), so the BE can still fetch splits during DoGet. + // See #62259. + if (!deferredForArrowFlight) { + coordBase.close(); + } } } @@ -1648,9 +1690,12 @@ private void deleteExistingOutfileFilesInFe(OutFileClause outFileClause) throws "delete_existing_files requires a remote outfile sink"); Preconditions.checkState(outFileClause.getBrokerDesc().storageType() != StorageType.LOCAL, "delete_existing_files is not supported for local outfile sinks"); + // Concrete filesystems only accept their native schemes; normalize legacy compatibility + // schemes (e.g. cos:// with s3.* properties) before crossing the plugin boundary. + String filePath = outFileClause.getBrokerDesc().getFileLocation(outFileClause.getFilePath()); try (org.apache.doris.filesystem.FileSystem fs = FileSystemFactory.getFileSystem(outFileClause.getBrokerDesc())) { - fs.delete(Location.of(FileSystemUtil.extractParentDirectory(outFileClause.getFilePath())), true); + fs.delete(Location.of(FileSystemUtil.extractParentDirectory(filePath)), true); } catch (java.io.IOException e) { throw new UserException("Failed to delete existing files: " + e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index f76b687e77fb00..6a5695cebb040f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -40,6 +40,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Table; @@ -1152,61 +1153,97 @@ public TFetchResourceResult fetchResource() throws TException { @Override public TMasterOpResult forward(TMasterOpRequest params) throws TException { + validateForwardRequester(params); + TMasterOpResult shortcut = handleForwardShortcut(params); + if (shortcut != null) { + return shortcut; + } + logForwardRequest(params); + ConnectContext context = createForwardContext(params); + ConnectProcessor processor = createForwardProcessor(context); + Runnable clearCallback = registerProxyQuery(params, context); + try { + return executeForward(params, context, processor); + } finally { + ConnectContext.remove(); + clearCallback.run(); + } + } + + private void validateForwardRequester(TMasterOpRequest params) throws TException { Frontend fe = Env.getCurrentEnv().checkFeExist(params.getClientNodeHost(), params.getClientNodePort()); - if (fe == null) { - LOG.warn("reject request from invalid host. client: {}", params.getClientNodeHost()); - throw new TException("request from invalid host was rejected."); + if (fe != null) { + return; } + LOG.warn("reject request from invalid host. client: {}", params.getClientNodeHost()); + throw new TException("request from invalid host was rejected."); + } + + private TMasterOpResult handleForwardShortcut(TMasterOpRequest params) throws TException { if (params.isSyncJournalOnly()) { - final TMasterOpResult result = new TMasterOpResult(); - result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); - // just make the protocol happy - result.setPacket("".getBytes()); - return result; + return createForwardResultWithJournalSync(); } if (params.getGroupCommitInfo() != null && params.getGroupCommitInfo().isGetGroupCommitLoadBeId()) { - final TGroupCommitInfo info = params.getGroupCommitInfo(); - final TMasterOpResult result = new TMasterOpResult(); - try { - result.setGroupCommitLoadBeId(Env.getCurrentEnv().getGroupCommitManager() - .selectBackendForGroupCommitInternal(info.groupCommitLoadTableId, info.cluster)); - } catch (LoadException | DdlException e) { - throw new TException(e.getMessage()); - } - // just make the protocol happy - result.setPacket("".getBytes()); - return result; + return handleGroupCommitLoadBeId(params.getGroupCommitInfo()); } if (params.getGroupCommitInfo() != null && params.getGroupCommitInfo().isUpdateLoadData()) { - final TGroupCommitInfo info = params.getGroupCommitInfo(); - final TMasterOpResult result = new TMasterOpResult(); Env.getCurrentEnv().getGroupCommitManager() - .updateLoadData(info.tableId, info.receiveData); - // just make the protocol happy - result.setPacket("".getBytes()); - return result; + .updateLoadData(params.getGroupCommitInfo().tableId, params.getGroupCommitInfo().receiveData); + return createForwardResultWithoutJournalSync(); } - if (params.isSetCancelQeury() && params.isCancelQeury()) { - if (!params.isSetQueryId()) { - throw new TException("a query id is needed to cancel a query"); - } - TUniqueId queryId = params.getQueryId(); - ConnectContext ctx = proxyQueryIdToConnCtx.get(queryId); - if (ctx != null) { - ctx.cancelQuery(new Status(TStatusCode.CANCELLED, "cancel query by forward request.")); - } - final TMasterOpResult result = new TMasterOpResult(); - result.setStatusCode(0); - result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); - // just make the protocol happy - result.setPacket("".getBytes()); - return result; + if (!params.isSetCancelQeury() || !params.isCancelQeury()) { + return null; + } + return handleForwardCancel(params); + } + + private TMasterOpResult createForwardResultWithJournalSync() { + TMasterOpResult result = new TMasterOpResult(); + result.setMaxJournalId(Env.getCurrentEnv().getMaxJournalId()); + result.setPacket("".getBytes()); + return result; + } + + private TMasterOpResult createForwardResultWithoutJournalSync() { + TMasterOpResult result = new TMasterOpResult(); + // Group commit shortcuts update master memory without producing a journal id. Say so explicitly + // instead of relying on the default value of the required maxJournalId field. + result.setMaxJournalId(0L); + result.setPacket("".getBytes()); + return result; + } + + private TMasterOpResult handleGroupCommitLoadBeId(TGroupCommitInfo info) throws TException { + TMasterOpResult result = createForwardResultWithoutJournalSync(); + try { + result.setGroupCommitLoadBeId(Env.getCurrentEnv().getGroupCommitManager() + .selectBackendForGroupCommitInternal(info.groupCommitLoadTableId, info.cluster)); + } catch (LoadException | DdlException e) { + throw new TException(e.getMessage()); + } + return result; + } + + private TMasterOpResult handleForwardCancel(TMasterOpRequest params) throws TException { + if (!params.isSetQueryId()) { + throw new TException("a query id is needed to cancel a query"); + } + ConnectContext context = proxyQueryIdToConnCtx.get(params.getQueryId()); + if (context != null) { + context.cancelQuery(new Status(TStatusCode.CANCELLED, "cancel query by forward request.")); } + TMasterOpResult result = createForwardResultWithJournalSync(); + result.setStatusCode(0); + return result; + } - // add this log so that we can track this stmt + private void logForwardRequest(TMasterOpRequest params) { if (LOG.isDebugEnabled()) { LOG.debug("receive forwarded stmt {} from FE: {}", params.getStmtId(), params.getClientNodeHost()); } + } + + private ConnectContext createForwardContext(TMasterOpRequest params) { ConnectContext context = new ConnectContext(null, true, params.getSessionId()); // Set current connected FE to the client address, so that we can know where // this request come from. @@ -1214,28 +1251,35 @@ public TMasterOpResult forward(TMasterOpRequest params) throws TException { if (Config.isCloudMode() && !Strings.isNullOrEmpty(params.getCloudCluster())) { context.setCloudCluster(params.getCloudCluster()); } + return context; + } - ConnectProcessor processor = null; + private ConnectProcessor createForwardProcessor(ConnectContext context) throws TException { if (context.getConnectType().equals(ConnectType.MYSQL)) { - processor = new MysqlConnectProcessor(context); - } else if (context.getConnectType().equals(ConnectType.ARROW_FLIGHT_SQL)) { - processor = new FlightSqlConnectProcessor(context); - } else { - throw new TException("unknown ConnectType: " + context.getConnectType()); + return new MysqlConnectProcessor(context); + } + if (context.getConnectType().equals(ConnectType.ARROW_FLIGHT_SQL)) { + return new FlightSqlConnectProcessor(context); } - Runnable clearCallback = () -> {}; - if (params.isSetQueryId()) { - proxyQueryIdToConnCtx.put(params.getQueryId(), context); - clearCallback = () -> proxyQueryIdToConnCtx.remove(params.getQueryId()); + throw new TException("unknown ConnectType: " + context.getConnectType()); + } + + private Runnable registerProxyQuery(TMasterOpRequest params, ConnectContext context) { + if (!params.isSetQueryId()) { + return () -> {}; } + proxyQueryIdToConnCtx.put(params.getQueryId(), context); + return () -> proxyQueryIdToConnCtx.remove(params.getQueryId()); + } + + private TMasterOpResult executeForward(TMasterOpRequest params, ConnectContext context, + ConnectProcessor processor) throws TException { TMasterOpResult result = processor.proxyExecute(params); if (QueryState.MysqlStateType.ERR.name().equalsIgnoreCase(result.getStatus())) { context.getState().setError(result.getStatus()); - } else { - context.getState().setOk(); + return result; } - ConnectContext.remove(); - clearCallback.run(); + context.getState().setOk(); return result; } @@ -3284,6 +3328,7 @@ public TFrontendPingFrontendResult ping(TFrontendPingFrontendRequest request) th result.setRpcPort(Config.rpc_port); result.setArrowFlightSqlPort(Config.arrow_flight_sql_port); result.setVersion(Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH); + result.setLocalResourceGroup(Config.local_resource_group); result.setLastStartupTime(exeEnv.getStartupTime()); result.setProcessUUID(exeEnv.getProcessUUID()); if (exeEnv.getDiskInfos() != null) { @@ -4658,26 +4703,6 @@ public TCreatePartitionResult createPartition(TCreatePartitionRequest request) t } } - // check partition's number limit. because partitions in addPartitionClauseMap may be duplicated with existing - // partitions, which would lead to false positive. so we should check the partition number AFTER adding new - // partitions using its ACTUAL NUMBER, rather than the sum of existing and requested partitions. - int partitionNum = olapTable.getPartitionNum(); - int autoPartitionLimit = Config.max_auto_partition_num; - if (partitionNum > autoPartitionLimit) { - String errorMessage = String.format( - "partition numbers %d exceeded limit of variable max_auto_partition_num %d", - partitionNum, autoPartitionLimit); - LOG.warn(errorMessage); - errorStatus.setErrorMsgs(Lists.newArrayList(errorMessage)); - result.setStatus(errorStatus); - LOG.warn("send create partition error status: {}", result); - return result; - } else if (partitionNum > autoPartitionLimit * 8 / 10) { - LOG.warn("Table {}.{} auto partition count {} is approaching limit {} (>80%)." - + " Consider increasing max_auto_partition_num.", - db.getFullName(), olapTable.getName(), partitionNum, autoPartitionLimit); - } - // build partition & tablets List tablets = new ArrayList<>(); List slaveTablets = new ArrayList<>(); @@ -4689,41 +4714,59 @@ public TCreatePartitionResult createPartition(TCreatePartitionRequest request) t && request.isEnableAdaptiveRandomBucket(); boolean loadToSingleTablet = request.isSetLoadToSingleTablet() && request.isLoadToSingleTablet(); final boolean hasBeEndpoint = request.isSetBeEndpoint(); - // Lazy: resolved on the first CloudTablet that needs it (skipped on cache-hit). - String cachedClusterId = null; - for (String partitionName : addPartitionClauseMap.keySet()) { - Partition partition = table.getPartition(partitionName); - // For thread safety, we preserve the tablet distribution information of each partition - // before calling getOrSetAutoPartitionInfo, but not check the partition first - List partitionTablets = new ArrayList<>(); - List partitionSlaveTablets = new ArrayList<>(); - TOlapTablePartition tPartition = new TOlapTablePartition(); - tPartition.setId(partition.getId()); - int partColNum = partitionInfo.getPartitionColumns().size(); + List partitionSnapshots = new ArrayList<>(); + + olapTable.readLock(); + try { + // check partition's number limit. because partitions in addPartitionClauseMap may be duplicated with + // existing partitions, which would lead to false positive. so we should check the partition number AFTER + // adding new partitions using its ACTUAL NUMBER, rather than the sum of existing and requested partitions. + int partitionNum = olapTable.getPartitionNum(); + int autoPartitionLimit = Config.max_auto_partition_num; + if (partitionNum > autoPartitionLimit) { + String errorMessage = String.format( + "partition numbers %d exceeded limit of variable max_auto_partition_num %d", + partitionNum, autoPartitionLimit); + LOG.warn(errorMessage); + errorStatus.setErrorMsgs(Lists.newArrayList(errorMessage)); + result.setStatus(errorStatus); + LOG.warn("send create partition error status: {}", result); + return result; + } else if (partitionNum > autoPartitionLimit * 8 / 10) { + LOG.warn("Table {}.{} auto partition count {} is approaching limit {} (>80%)." + + " Consider increasing max_auto_partition_num.", + db.getFullName(), olapTable.getName(), partitionNum, autoPartitionLimit); + } + try { - OlapTableSink.setPartitionKeys(tPartition, partitionInfo.getItem(partition.getId()), partColNum); + partitionSnapshots.addAll(snapshotPartitionResultsByName(olapTable, addPartitionClauseMap.keySet(), + loadToSingleTablet, enableAdaptiveRandomBucket, "auto partition")); } catch (UserException ex) { errorStatus.setErrorMsgs(Lists.newArrayList(ex.getMessage())); result.setStatus(errorStatus); LOG.warn("send create partition error status: {}", result); return result; } - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { - tPartition.addToIndexes(new TOlapTableIndexTablets(index.getId(), Lists.newArrayList( - index.getTablets().stream().map(Tablet::getId).collect(Collectors.toList())))); - tPartition.setNumBuckets(index.getTablets().size()); - } - tPartition.setIsMutable(olapTable.getPartitionInfo().getIsMutable(partition.getId())); - boolean randomDistribution = - partition.getDistributionInfo().getType() == DistributionInfo.DistributionInfoType.RANDOM; - boolean cacheLoadTabletIdx = - (loadToSingleTablet || enableAdaptiveRandomBucket) && randomDistribution; + } finally { + olapTable.readUnlock(); + } + + // Lazy: resolved on the first CloudTablet that needs it (skipped on cache-hit). + String cachedClusterId = null; + for (PartitionResultSnapshot partitionSnapshot : partitionSnapshots) { + Partition partition = partitionSnapshot.partition; + long partitionId = partitionSnapshot.partitionId; + TOlapTablePartition tPartition = partitionSnapshot.tPartition; + boolean cacheLoadTabletIdx = partitionSnapshot.cacheLoadTabletIdx; partitions.add(tPartition); - // tablet + // For thread safety, we preserve the tablet distribution information of each partition + // before calling getOrSetAutoPartitionInfo, but not check the partition first + List partitionTablets = new ArrayList<>(); + List partitionSlaveTablets = new ArrayList<>(); AtomicLong cachedLoadTabletIdx = new AtomicLong(-1); if (needUseCache && Env.getCurrentGlobalTransactionMgr().getAutoPartitionCacheMgr() - .getAutoPartitionInfo(txnId, partition.getId(), partitionTablets, + .getAutoPartitionInfo(txnId, partitionId, partitionTablets, partitionSlaveTablets, cachedLoadTabletIdx)) { if (cacheLoadTabletIdx) { tPartition.setLoadTabletIdx(cachedLoadTabletIdx.get()); @@ -4747,52 +4790,49 @@ public TCreatePartitionResult createPartition(TCreatePartitionRequest request) t return result; } } - int quorum = olapTable.getPartitionInfo().getReplicaAllocation(partition.getId()).getTotalReplicaNum() / 2 - + 1; - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { - for (Tablet tablet : index.getTablets()) { - // we should ensure the replica backend is alive - // otherwise, there will be a 'unknown node id, id=xxx' error for stream load - // BE id -> path hash - Multimap bePathsMap; - try { - if (tablet instanceof CloudTablet) { - CloudTablet cloudTablet = (CloudTablet) tablet; - if (hasBeEndpoint) { - bePathsMap = cloudTablet.getNormalReplicaBackendPathMap(request.be_endpoint); - } else { - if (cachedClusterId == null) { - cachedClusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCurrentClusterId(); - } - bePathsMap = cloudTablet.getNormalReplicaBackendPathMapByClusterId(cachedClusterId); - } + int quorum = partitionSnapshot.quorum; + for (Tablet tablet : partitionSnapshot.tablets) { + // we should ensure the replica backend is alive + // otherwise, there will be a 'unknown node id, id=xxx' error for stream load + // BE id -> path hash + Multimap bePathsMap; + try { + if (tablet instanceof CloudTablet) { + CloudTablet cloudTablet = (CloudTablet) tablet; + if (hasBeEndpoint) { + bePathsMap = cloudTablet.getNormalReplicaBackendPathMap(request.be_endpoint); } else { - bePathsMap = tablet.getNormalReplicaBackendPathMap(); + if (cachedClusterId == null) { + cachedClusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCurrentClusterId(); + } + bePathsMap = cloudTablet.getNormalReplicaBackendPathMapByClusterId(cachedClusterId); } - } catch (UserException ex) { - errorStatus.setErrorMsgs(Lists.newArrayList(ex.getMessage())); - result.setStatus(errorStatus); - LOG.warn("send create partition error status: {}", result); - return result; - } - if (bePathsMap.keySet().size() < quorum) { - LOG.warn("auto go quorum exception"); - } - if (request.isSetWriteSingleReplica() && request.isWriteSingleReplica()) { - Long[] nodes = bePathsMap.keySet().toArray(new Long[0]); - Random random = new SecureRandom(); - Long masterNode = nodes[random.nextInt(nodes.length)]; - Multimap slaveBePathsMap = bePathsMap; - slaveBePathsMap.removeAll(masterNode); - partitionTablets.add(new TTabletLocation(tablet.getId(), - Lists.newArrayList(Sets.newHashSet(masterNode)))); - partitionSlaveTablets.add(new TTabletLocation(tablet.getId(), - Lists.newArrayList(slaveBePathsMap.keySet()))); } else { - partitionTablets.add(new TTabletLocation(tablet.getId(), - Lists.newArrayList(bePathsMap.keySet()))); + bePathsMap = tablet.getNormalReplicaBackendPathMap(); } + } catch (UserException ex) { + errorStatus.setErrorMsgs(Lists.newArrayList(ex.getMessage())); + result.setStatus(errorStatus); + LOG.warn("send create partition error status: {}", result); + return result; + } + if (bePathsMap.keySet().size() < quorum) { + LOG.warn("auto go quorum exception"); + } + if (request.isSetWriteSingleReplica() && request.isWriteSingleReplica()) { + Long[] nodes = bePathsMap.keySet().toArray(new Long[0]); + Random random = new SecureRandom(); + Long masterNode = nodes[random.nextInt(nodes.length)]; + Multimap slaveBePathsMap = bePathsMap; + slaveBePathsMap.removeAll(masterNode); + partitionTablets.add(new TTabletLocation(tablet.getId(), + Lists.newArrayList(Sets.newHashSet(masterNode)))); + partitionSlaveTablets.add(new TTabletLocation(tablet.getId(), + Lists.newArrayList(slaveBePathsMap.keySet()))); + } else { + partitionTablets.add(new TTabletLocation(tablet.getId(), + Lists.newArrayList(bePathsMap.keySet()))); } } @@ -4820,7 +4860,7 @@ public TCreatePartitionResult createPartition(TCreatePartitionRequest request) t if (needUseCache) { long loadTabletIdx = cacheLoadTabletIdx ? tPartition.getLoadTabletIdx() : -1; long cachedTabletIdx = Env.getCurrentGlobalTransactionMgr().getAutoPartitionCacheMgr() - .getOrSetAutoPartitionInfo(txnId, partition.getId(), partitionTablets, + .getOrSetAutoPartitionInfo(txnId, partitionId, partitionTablets, partitionSlaveTablets, loadTabletIdx); if (cacheLoadTabletIdx) { tPartition.setLoadTabletIdx(cachedTabletIdx); @@ -4943,6 +4983,14 @@ public TReplacePartitionResult replacePartition(TReplacePartitionRequest request } } + Backend requestBackend = request.isSetBeEndpoint() ? resolveBeEndpoint(request.getBeEndpoint()) : null; + long adaptiveBucketBeId = requestBackend != null ? requestBackend.getId() : -1L; + TUniqueId queryId = request.isSetQueryId() ? request.getQueryId() : null; + boolean enableAdaptiveRandomBucket = request.isSetEnableAdaptiveRandomBucket() + && request.isEnableAdaptiveRandomBucket(); + boolean loadToSingleTablet = request.isSetLoadToSingleTablet() && request.isLoadToSingleTablet(); + final boolean replaceHasBeEndpoint = request.isSetBeEndpoint(); + InsertOverwriteManager overwriteManager = Env.getCurrentEnv().getInsertOverwriteManager(); ReentrantLock taskLock = overwriteManager.getLock(taskGroupId); if (taskLock == null) { @@ -4957,6 +5005,7 @@ public TReplacePartitionResult replacePartition(TReplacePartitionRequest request ArrayList pendingPartitionIds = new ArrayList<>(); // pending: [1 2] ArrayList newPartitionIds = new ArrayList<>(); // requested temp partition ids. for [7 8] boolean needReplace = false; + List partitionSnapshots = new ArrayList<>(); try { taskLock.lock(); // double check lock. maybe taskLock is not null, but has been removed from the Map. means the task failed. @@ -5004,29 +5053,37 @@ public TReplacePartitionResult replacePartition(TReplacePartitionRequest request } } } - } catch (DdlException | RuntimeException ex) { - errorStatus.setErrorMsgs(Lists.newArrayList(ex.getMessage())); - result.setStatus(errorStatus); - LOG.warn("send create partition error status: {}", result); - return result; - } finally { - taskLock.unlock(); - } - // result: [1 2 5 6], make it [7 8 5 6] - int idx = 0; - if (needReplace) { - for (int i = 0; i < reqPartitionIds.size(); i++) { - if (reqPartitionIds.get(i).equals(resultPartitionIds.get(i))) { - resultPartitionIds.set(i, newPartitionIds.get(idx++)); + // result: [1 2 5 6], make it [7 8 5 6] + int idx = 0; + if (needReplace) { + for (int i = 0; i < reqPartitionIds.size(); i++) { + if (reqPartitionIds.get(i).equals(resultPartitionIds.get(i))) { + resultPartitionIds.set(i, newPartitionIds.get(idx++)); + } } } - } - if (idx != newPartitionIds.size()) { - errorStatus.addToErrorMsgs("changed partition number " + idx + " is not correct"); + if (idx != newPartitionIds.size()) { + errorStatus.addToErrorMsgs("changed partition number " + idx + " is not correct"); + result.setStatus(errorStatus); + LOG.warn("send create partition error status: {}", result); + return result; + } + + olapTable.readLock(); + try { + partitionSnapshots.addAll(snapshotPartitionResultsById(olapTable, resultPartitionIds, + loadToSingleTablet, enableAdaptiveRandomBucket, "replace partition")); + } finally { + olapTable.readUnlock(); + } + } catch (UserException | RuntimeException ex) { + errorStatus.setErrorMsgs(Lists.newArrayList(ex.getMessage())); result.setStatus(errorStatus); LOG.warn("send create partition error status: {}", result); return result; + } finally { + taskLock.unlock(); } if (LOG.isDebugEnabled()) { @@ -5043,51 +5100,23 @@ public TReplacePartitionResult replacePartition(TReplacePartitionRequest request List partitions = new ArrayList<>(); List tablets = new ArrayList<>(); List slaveTablets = new ArrayList<>(); - PartitionInfo partitionInfo = olapTable.getPartitionInfo(); - Backend requestBackend = request.isSetBeEndpoint() ? resolveBeEndpoint(request.getBeEndpoint()) : null; - long adaptiveBucketBeId = requestBackend != null ? requestBackend.getId() : -1L; - TUniqueId queryId = request.isSetQueryId() ? request.getQueryId() : null; - boolean enableAdaptiveRandomBucket = request.isSetEnableAdaptiveRandomBucket() - && request.isEnableAdaptiveRandomBucket(); - boolean loadToSingleTablet = request.isSetLoadToSingleTablet() && request.isLoadToSingleTablet(); - final boolean replaceHasBeEndpoint = request.isSetBeEndpoint(); // Lazy: resolved on the first CloudTablet that needs it. String replaceCachedClusterId = null; - for (long partitionId : resultPartitionIds) { - Partition partition = olapTable.getPartition(partitionId); + for (PartitionResultSnapshot partitionSnapshot : partitionSnapshots) { + Partition partition = partitionSnapshot.partition; + long partitionId = partitionSnapshot.partitionId; + TOlapTablePartition tPartition = partitionSnapshot.tPartition; + boolean cacheLoadTabletIdx = partitionSnapshot.cacheLoadTabletIdx; + partitions.add(tPartition); // For thread safety, we preserve the tablet distribution information of each partition // before calling getOrSetAutoPartitionInfo, but not check the partition first List partitionTablets = new ArrayList<>(); List partitionSlaveTablets = new ArrayList<>(); - TOlapTablePartition tPartition = new TOlapTablePartition(); - tPartition.setId(partition.getId()); - - // set partition keys - int partColNum = partitionInfo.getPartitionColumns().size(); - try { - OlapTableSink.setPartitionKeys(tPartition, partitionInfo.getItem(partition.getId()), partColNum); - } catch (UserException ex) { - errorStatus.setErrorMsgs(Lists.newArrayList(ex.getMessage())); - result.setStatus(errorStatus); - LOG.warn("send replace partition error status: {}", result); - return result; - } - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { - tPartition.addToIndexes(new TOlapTableIndexTablets(index.getId(), Lists.newArrayList( - index.getTablets().stream().map(Tablet::getId).collect(Collectors.toList())))); - tPartition.setNumBuckets(index.getTablets().size()); - } - tPartition.setIsMutable(olapTable.getPartitionInfo().getIsMutable(partition.getId())); - boolean randomDistribution = - partition.getDistributionInfo().getType() == DistributionInfo.DistributionInfoType.RANDOM; - boolean cacheLoadTabletIdx = - (loadToSingleTablet || enableAdaptiveRandomBucket) && randomDistribution; - partitions.add(tPartition); // tablet AtomicLong cachedLoadTabletIdx = new AtomicLong(-1); if (needUseCache && txnId != 0 && Env.getCurrentGlobalTransactionMgr().getAutoPartitionCacheMgr() - .getAutoPartitionInfo(txnId, partition.getId(), partitionTablets, + .getAutoPartitionInfo(txnId, partitionId, partitionTablets, partitionSlaveTablets, cachedLoadTabletIdx)) { if (cacheLoadTabletIdx) { tPartition.setLoadTabletIdx(cachedLoadTabletIdx.get()); @@ -5111,53 +5140,50 @@ public TReplacePartitionResult replacePartition(TReplacePartitionRequest request return result; } } - int quorum = olapTable.getPartitionInfo().getReplicaAllocation(partition.getId()).getTotalReplicaNum() / 2 - + 1; - for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { - for (Tablet tablet : index.getTablets()) { - // we should ensure the replica backend is alive - // otherwise, there will be a 'unknown node id, id=xxx' error for stream load - // BE id -> path hash - Multimap bePathsMap; - try { - if (tablet instanceof CloudTablet) { - CloudTablet cloudTablet = (CloudTablet) tablet; - if (replaceHasBeEndpoint) { - bePathsMap = cloudTablet.getNormalReplicaBackendPathMap(request.be_endpoint); - } else { - if (replaceCachedClusterId == null) { - replaceCachedClusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCurrentClusterId(); - } - bePathsMap = cloudTablet - .getNormalReplicaBackendPathMapByClusterId(replaceCachedClusterId); - } + int quorum = partitionSnapshot.quorum; + for (Tablet tablet : partitionSnapshot.tablets) { + // we should ensure the replica backend is alive + // otherwise, there will be a 'unknown node id, id=xxx' error for stream load + // BE id -> path hash + Multimap bePathsMap; + try { + if (tablet instanceof CloudTablet) { + CloudTablet cloudTablet = (CloudTablet) tablet; + if (replaceHasBeEndpoint) { + bePathsMap = cloudTablet.getNormalReplicaBackendPathMap(request.be_endpoint); } else { - bePathsMap = tablet.getNormalReplicaBackendPathMap(); + if (replaceCachedClusterId == null) { + replaceCachedClusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCurrentClusterId(); + } + bePathsMap = cloudTablet + .getNormalReplicaBackendPathMapByClusterId(replaceCachedClusterId); } - } catch (UserException ex) { - errorStatus.setErrorMsgs(Lists.newArrayList(ex.getMessage())); - result.setStatus(errorStatus); - LOG.warn("send replace partition error status: {}", result); - return result; - } - if (bePathsMap.keySet().size() < quorum) { - LOG.warn("auto go quorum exception"); - } - if (request.isSetWriteSingleReplica() && request.isWriteSingleReplica()) { - Long[] nodes = bePathsMap.keySet().toArray(new Long[0]); - Random random = new SecureRandom(); - Long masterNode = nodes[random.nextInt(nodes.length)]; - Multimap slaveBePathsMap = bePathsMap; - slaveBePathsMap.removeAll(masterNode); - partitionTablets.add(new TTabletLocation(tablet.getId(), - Lists.newArrayList(Sets.newHashSet(masterNode)))); - partitionSlaveTablets.add(new TTabletLocation(tablet.getId(), - Lists.newArrayList(slaveBePathsMap.keySet()))); } else { - partitionTablets.add(new TTabletLocation(tablet.getId(), - Lists.newArrayList(bePathsMap.keySet()))); + bePathsMap = tablet.getNormalReplicaBackendPathMap(); } + } catch (UserException ex) { + errorStatus.setErrorMsgs(Lists.newArrayList(ex.getMessage())); + result.setStatus(errorStatus); + LOG.warn("send replace partition error status: {}", result); + return result; + } + if (bePathsMap.keySet().size() < quorum) { + LOG.warn("auto go quorum exception"); + } + if (request.isSetWriteSingleReplica() && request.isWriteSingleReplica()) { + Long[] nodes = bePathsMap.keySet().toArray(new Long[0]); + Random random = new SecureRandom(); + Long masterNode = nodes[random.nextInt(nodes.length)]; + Multimap slaveBePathsMap = bePathsMap; + slaveBePathsMap.removeAll(masterNode); + partitionTablets.add(new TTabletLocation(tablet.getId(), + Lists.newArrayList(Sets.newHashSet(masterNode)))); + partitionSlaveTablets.add(new TTabletLocation(tablet.getId(), + Lists.newArrayList(slaveBePathsMap.keySet()))); + } else { + partitionTablets.add(new TTabletLocation(tablet.getId(), + Lists.newArrayList(bePathsMap.keySet()))); } } @@ -5189,14 +5215,14 @@ public TReplacePartitionResult replacePartition(TReplacePartitionRequest request if (needUseCache) { long loadTabletIdx = cacheLoadTabletIdx ? tPartition.getLoadTabletIdx() : -1; long cachedTabletIdx = Env.getCurrentGlobalTransactionMgr().getAutoPartitionCacheMgr() - .getOrSetAutoPartitionInfo(txnId, partition.getId(), partitionTablets, + .getOrSetAutoPartitionInfo(txnId, partitionId, partitionTablets, partitionSlaveTablets, loadTabletIdx); if (cacheLoadTabletIdx) { tPartition.setLoadTabletIdx(cachedTabletIdx); } if (LOG.isDebugEnabled()) { LOG.debug("Cache auto partition info, txnId: {}, partitionId: {}, " - + "tablets: {}, slaveTablets: {}", txnId, partition.getId(), + + "tablets: {}, slaveTablets: {}", txnId, partitionId, partitionTablets.size(), partitionSlaveTablets.size()); } } @@ -5228,6 +5254,94 @@ public TReplacePartitionResult replacePartition(TReplacePartitionRequest request return result; } + private static final class PartitionResultSnapshot { + private final Partition partition; + private final long partitionId; + private final TOlapTablePartition tPartition; + private final List tablets; + private final int quorum; + private final boolean cacheLoadTabletIdx; + + private PartitionResultSnapshot(Partition partition, long partitionId, + TOlapTablePartition tPartition, List tablets, int quorum, boolean cacheLoadTabletIdx) { + this.partition = partition; + this.partitionId = partitionId; + this.tPartition = tPartition; + this.tablets = tablets; + this.quorum = quorum; + this.cacheLoadTabletIdx = cacheLoadTabletIdx; + } + } + + private static List snapshotPartitionResultsByName(OlapTable olapTable, + Collection partitionNames, boolean loadToSingleTablet, boolean enableAdaptiveRandomBucket, + String resultName) throws UserException { + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + int partColNum = partitionInfo.getPartitionColumns().size(); + List partitionSnapshots = new ArrayList<>(); + for (String partitionName : partitionNames) { + Partition partition = olapTable.getPartition(partitionName); + if (partition == null) { + throw new UserException(String.format( + "partition %s was dropped concurrently while building %s result, please retry", + partitionName, resultName)); + } + partitionSnapshots.add(snapshotPartitionResult(partitionInfo, partColNum, partition, partitionName, + loadToSingleTablet, enableAdaptiveRandomBucket, resultName)); + } + return partitionSnapshots; + } + + private static List snapshotPartitionResultsById(OlapTable olapTable, + Collection partitionIds, boolean loadToSingleTablet, boolean enableAdaptiveRandomBucket, + String resultName) throws UserException { + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + int partColNum = partitionInfo.getPartitionColumns().size(); + List partitionSnapshots = new ArrayList<>(); + for (long partitionId : partitionIds) { + Partition partition = olapTable.getPartition(partitionId); + if (partition == null) { + throw new UserException(String.format( + "partition %d was dropped concurrently while building %s result, please retry", + partitionId, resultName)); + } + partitionSnapshots.add(snapshotPartitionResult(partitionInfo, partColNum, partition, + String.valueOf(partitionId), loadToSingleTablet, enableAdaptiveRandomBucket, resultName)); + } + return partitionSnapshots; + } + + private static PartitionResultSnapshot snapshotPartitionResult(PartitionInfo partitionInfo, int partColNum, + Partition partition, String partitionLabel, boolean loadToSingleTablet, + boolean enableAdaptiveRandomBucket, String resultName) throws UserException { + long partitionId = partition.getId(); + PartitionItem partitionItem = partitionInfo.getItem(partitionId); + if (partitionItem == null) { + throw new UserException(String.format( + "partition item of %s was dropped concurrently while building %s result, please retry", + partitionLabel, resultName)); + } + + TOlapTablePartition tPartition = new TOlapTablePartition(); + tPartition.setId(partitionId); + OlapTableSink.setPartitionKeys(tPartition, partitionItem, partColNum); + List partitionTabletSnapshot = new ArrayList<>(); + for (MaterializedIndex index : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) { + List indexTablets = new ArrayList<>(index.getTablets()); + tPartition.addToIndexes(new TOlapTableIndexTablets(index.getId(), Lists.newArrayList( + indexTablets.stream().map(Tablet::getId).collect(Collectors.toList())))); + tPartition.setNumBuckets(indexTablets.size()); + partitionTabletSnapshot.addAll(indexTablets); + } + tPartition.setIsMutable(partitionInfo.getIsMutable(partitionId)); + boolean randomDistribution = + partition.getDistributionInfo().getType() == DistributionInfo.DistributionInfoType.RANDOM; + boolean cacheLoadTabletIdx = (loadToSingleTablet || enableAdaptiveRandomBucket) && randomDistribution; + int quorum = partitionInfo.getReplicaAllocation(partitionId).getTotalReplicaNum() / 2 + 1; + return new PartitionResultSnapshot(partition, partitionId, tPartition, partitionTabletSnapshot, quorum, + cacheLoadTabletIdx); + } + public TGetMetaResult getMeta(TGetMetaRequest request) throws TException { String clientAddr = getClientAddrAsString(); if (LOG.isDebugEnabled()) { @@ -5582,6 +5696,7 @@ public TFetchRoutineLoadJobResult fetchRoutineLoadJob(TFetchRoutineLoadJobReques jobInfo.setLag(job.getLag()); jobInfo.setReasonOfStateChanged(job.getStateReason()); jobInfo.setErrorLogUrls(Joiner.on(", ").join(job.getErrorLogUrls())); + jobInfo.setFirstErrorMsg(job.getFirstErrorMsg()); jobInfo.setUserName(job.getUserIdentity().getQualifiedUser()); jobInfo.setCurrentAbortTaskNum(job.getJobStatistic().currentAbortedTaskNum); jobInfo.setIsAbnormalPause(job.isAbnormalPause()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java index ebc3347f07f27d..e64690a54cb10f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java @@ -189,6 +189,10 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con try { Preconditions.checkState(null != connectContext); Preconditions.checkState(!query.isEmpty()); + // Finalize the previous query's coordinator on this connection whose close was + // deferred (Arrow Flight keeps it alive across GetFlightInfo -> DoGet so the BE can + // fetch external-table splits during DoGet). By now the previous DoGet is done. #62259 + connectContext.closeFlightSqlDeferredExecutors(); // After the previous query was executed, there was no getStreamStatement to take away the result. connectContext.getFlightSqlChannel().reset(); connectContext.clearFlightSqlEndpointsLocations(); @@ -280,6 +284,14 @@ private FlightInfo executeQueryStatement(String peerIdentity, ConnectContext con } } } catch (Throwable e) { + // GetFlightInfo failed (e.g. the BE Arrow schema fetch above timed out or returned an + // error) after this query's coordinator may already have been deferred during planning. + // No FlightInfo is returned, so no DoGet will ever pull this query's results; finalize + // the deferred coordinator now (releasing its external-table batch SplitSource, query + // queue slot and query registration) instead of leaking it until the next query starts + // or the connection is torn down. The previous query's deferred coordinator was already + // finalized at the top of this method, so this only closes this failed query. See #62259. + connectContext.closeFlightSqlDeferredExecutors(); String errMsg = "get flight info statement failed, " + e.getMessage() + ", " + Util.getRootCauseMessage(e) + ", error code: " + connectContext.getState().getErrorCode() + ", error msg: " + connectContext.getState().getErrorMessage(); @@ -356,14 +368,22 @@ public void createPreparedStatement(final ActionCreatePreparedStatementRequest r final ByteString handle = ByteString.copyFromUtf8(context.peerIdentity() + ":" + preparedStatementId); connectContext.addPreparedQuery(preparedStatementId, query); - VectorSchemaRoot emptyVectorSchemaRoot = new VectorSchemaRoot(new ArrayList<>(), new ArrayList<>()); - final Schema parameterSchema = emptyVectorSchemaRoot.getSchema(); + // Close the temporary VectorSchemaRoot after extracting its Schema, otherwise the + // off-heap buffers backing its vectors are leaked on every prepare (FE direct memory leak). + final Schema parameterSchema; + try (VectorSchemaRoot emptyVectorSchemaRoot = + new VectorSchemaRoot(new ArrayList<>(), new ArrayList<>())) { + parameterSchema = emptyVectorSchemaRoot.getSchema(); + } // TODO FE does not have the ability to convert root fragment output expr into arrow schema. // However, the metaData schema returned by createPreparedStatement is usually not used by the client, // but it cannot be empty, otherwise it will be mistaken by the client as an updata statement. // see: https://github.com/apache/arrow/issues/38911 - Schema metaData = connectContext.getFlightSqlChannel() - .createOneOneSchemaRoot("ResultMeta", "UNIMPLEMENTED").getSchema(); + final Schema metaData; + try (VectorSchemaRoot metaSchemaRoot = connectContext.getFlightSqlChannel() + .createOneOneSchemaRoot("ResultMeta", "UNIMPLEMENTED")) { + metaData = metaSchemaRoot.getSchema(); + } listener.onNext(new Result( Any.pack(buildCreatePreparedStatementResult(handle, parameterSchema, metaData)).toByteArray())); } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java index a63fab3e9608f7..aa69339fd31a2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlConnectProcessor.java @@ -196,11 +196,20 @@ public void fetchArrowFlightSchema(int timeoutMs) { @Override public void close() throws Exception { ctx.setCommand(MysqlCommand.COM_SLEEP); + // Executors whose results are pulled from the BE keep their coordinator alive past + // GetFlightInfo (registered as deferred executors on the ConnectContext) so the BE can + // still fetch external-table splits during DoGet. Do NOT finalize those here; they are + // finalized when the next query starts or the connection is torn down. Executors that are + // not deferred (local results, or a query that already failed) are finalized now. See #62259. for (StmtExecutor asynExecutor : returnResultFromRemoteExecutor) { - asynExecutor.finalizeQuery(); + if (!asynExecutor.isDeferredForArrowFlight()) { + asynExecutor.finalizeQuery(); + } } returnResultFromRemoteExecutor.clear(); - executor.finalizeQuery(); + if (executor != null && !executor.isDeferredForArrowFlight()) { + executor.finalizeQuery(); + } ctx.clear(); ConnectContext.remove(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java index 8ce5f72b4000eb..4f54132876b0c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/results/FlightSqlChannel.java @@ -156,6 +156,12 @@ public long resultNum() { return resultCache.size(); } + // Returns the off-heap memory (bytes) currently held by this channel's Arrow allocator. + // Exposed so tests can assert prepared-statement handling does not leak VectorSchemaRoot buffers. + public long getAllocatedMemory() { + return allocator.getAllocatedMemory(); + } + public void reset() { resultCache.invalidateAll(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java index 3002116b386aa7..743f5c49d44a60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgr.java @@ -55,6 +55,11 @@ public int registerConnection(ConnectContext ctx) { @Override public void unregisterConnection(ConnectContext ctx) { + // Finalize any Arrow Flight query whose coordinator was kept alive across the + // GetFlightInfo -> DoGet phases (see #62259), releasing its resources (e.g. external-table + // batch SplitSources and the query queue slot) when the connection is torn down: idle/query + // timeout, bearer token expiry, or explicit CloseSession all reach here. + ctx.closeFlightSqlDeferredExecutors(); ctx.closeTxn(); if (connectionMap.remove(ctx.getConnectionId()) != null) { numberConnection.decrementAndGet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java index 78ce44dc3aff76..41fae5ddb3393c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -34,42 +35,53 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; +import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveUnionAnchor; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; +import org.apache.doris.nereids.trees.plans.physical.PhysicalWorkTableReference; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.ImmutableSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; /** * Records column-level query-hit and filter-hit statistics from the Nereids physical plan. - * Called once per query in NereidsPlanner after plan translation. + * Only OlapTable scans are recorded; DML, EXPLAIN, and internal queries are skipped. * - *

queryHit: SELECT output columns (alias-unwrapped), GROUP BY keys, - * ORDER BY keys, window PARTITION BY / ORDER BY keys, aggregate input columns. - * filterHit: WHERE predicate columns and JOIN ON conditions. - * Only OlapTable scans are recorded. DML, EXPLAIN, and internal queries are skipped. - * Per query each table's count is incremented at most once regardless of scan count. + *

{@code exprIdToScan} holds only base scan-backed ExprIds; every derived slot (alias, + * aggregate output, CTE consumer, set-op output) is an edge in {@code derivedSlotInputs} + * pointing at its own inputs. {@link #resolveBaseScanExprIds} expands that edge map + * recursively, however many computed hops deep a slot's provenance chain is. */ public class QueryStatsRecorder { private static final Logger LOG = LogManager.getLogger(QueryStatsRecorder.class); @@ -99,11 +111,12 @@ public static void record(PhysicalPlan plan, StatementContext stmtContext) { } } } - } catch (Exception e) { + } catch (Throwable t) { + // Best-effort telemetry: catch Errors too (e.g. StackOverflowError), never break the query. ConnectContext cc = stmtContext.getConnectContext(); String queryId = (cc != null && cc.queryId() != null) ? cc.queryId().toString() : "unknown"; - LOG.warn("Failed to build query stats deltas for query={}", queryId, e); + LOG.warn("Failed to build query stats deltas for query={}", queryId, t); } } @@ -115,7 +128,10 @@ static Map collectDeltas(PhysicalPlan plan) { Map exprIdToScan = new HashMap<>(); Map exprIdToColName = new HashMap<>(); Map deltas = new HashMap<>(); - walkPlan(plan, exprIdToScan, exprIdToColName, deltas); + // ExprId -> input slots for any derived slot (alias, aggregate output, CTE consumer + // slot, set-operation output). resolveBaseScanExprIds expands this recursively. + Map> derivedSlotInputs = new HashMap<>(); + walkPlan(plan, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); if (exprIdToScan.isEmpty()) { return deltas; } @@ -125,20 +141,9 @@ static Map collectDeltas(PhysicalPlan plan) { : plan.getOutput(); for (NamedExpression ne : rootExprs) { SlotReference sr = unwrapAlias(ne); - if (sr == null) { - continue; - } - PhysicalOlapScan sourceScan = exprIdToScan.get(sr.getExprId()); - if (sourceScan == null) { - continue; - } - StatsDelta delta = getOrCreateDelta(deltas, sourceScan); - if (delta != null) { - String colName = sr.getOriginalColumn().map(col -> col.getName()) - .orElseGet(() -> exprIdToColName.get(sr.getExprId())); - if (colName != null) { - delta.addQueryStats(colName); - } + ExprId lookupId = (sr != null) ? sr.getExprId() : ne.getExprId(); + for (ExprId baseId : resolveBaseScanExprIds(lookupId, exprIdToScan, derivedSlotInputs)) { + recordExprIdAsQueryHit(baseId, exprIdToScan, exprIdToColName, deltas); } } return deltas; @@ -170,17 +175,14 @@ static boolean shouldRecord(StatementContext ctx) { } /** - * Single-pass tree walk: registers scan output slots into exprIdToScan, - * records filterHit for WHERE conjuncts, and records queryHit for - * GROUP BY / ORDER BY / window keys and aggregate input columns. - * Children are visited before the current node so scans are registered first. - * PhysicalLazyMaterializeOlapScan is checked before PhysicalOlapScan - * because it is a subclass; the inner scan's metadata must be used. + * Single-pass tree walk: registers scan slots, records filterHit for WHERE/JOIN conditions, + * and queryHit for GROUP BY / ORDER BY / window keys and aggregate inputs. */ private static void walkPlan(Plan plan, Map exprIdToScan, Map exprIdToColName, - Map deltas) { + Map deltas, + Map> derivedSlotInputs) { if (plan instanceof PhysicalStorageLayerAggregate) { // COUNT(*)/MIN/MAX pushdown — the aggregate wraps the real scan but has no children. PhysicalRelation inner = ((PhysicalStorageLayerAggregate) plan).getRelation(); @@ -216,22 +218,69 @@ private static void walkPlan(Plan plan, } return; } - // TODO: PhysicalCTEConsumer slots use consumer-side ExprIds that differ from the producer - // scan's ExprIds, so CTE column stats are silently missed. Fix requires mapping consumer - // slots back to producer slots via StatementContext.getConsumerToProducerSlotMap(). + // PhysicalCTEProducer: walk its single child so the producer scan is registered + // before any PhysicalCTEConsumer nodes are encountered during the tree walk. + // Uses plan.children() for consistency with the rest of walkPlan. + if (plan instanceof PhysicalCTEProducer) { + walkPlan(plan.children().get(0), + exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); + return; + } + // PhysicalCTEConsumer: link consumer slots to producer slots; resolveBaseScanExprIds + // expands through this whether the producer column is direct or itself computed. + if (plan instanceof PhysicalCTEConsumer) { + PhysicalCTEConsumer cteConsumer = (PhysicalCTEConsumer) plan; + for (Slot consumerSlot : cteConsumer.getOutput()) { + Slot producerSlot = cteConsumer.getProducerSlot(consumerSlot); + derivedSlotInputs.put(consumerSlot.getExprId(), ImmutableSet.of(producerSlot)); + } + } + // PhysicalRecursiveUnion extends PhysicalBinary (not PhysicalSetOperation): left() is the + // base case (wrapped in PhysicalRecursiveUnionAnchor), right() is the recursive case, which + // contains a PhysicalWorkTableReference re-scanning the base case's own output on every + // iteration. WorkTableReference's output slots get brand-new ExprIds (no named API back to + // the base case) that only positionally correspond to the base case's own output, scoped by + // matching CTEId to the anchor. Any Filter/Join predicate inside the recursive branch that + // references those slots is walked as part of walking right() below, so this linkage must + // be established in derivedSlotInputs in between walking left() and right() — the generic + // post-order recursion (walk all children, then run this node's own handler) can't do that, + // so PhysicalRecursiveUnion controls its own children's walk order here instead. + if (plan instanceof PhysicalRecursiveUnion) { + PhysicalRecursiveUnion recursiveUnion = (PhysicalRecursiveUnion) plan; + Plan baseCase = recursiveUnion.left(); + Plan recursiveCase = recursiveUnion.right(); + walkPlan(baseCase, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); + List> childrenOutputs = recursiveUnion.getRegularChildrenOutputs(); + if (baseCase instanceof PhysicalRecursiveUnionAnchor && !childrenOutputs.isEmpty()) { + CTEId anchorCteId = ((PhysicalRecursiveUnionAnchor) baseCase).getCteId(); + List baseOutput = childrenOutputs.get(0); + List workTables = recursiveCase.collectToList( + node -> node instanceof PhysicalWorkTableReference + && anchorCteId.equals(((PhysicalWorkTableReference) node).getCteId())); + for (PhysicalWorkTableReference workTable : workTables) { + List workTableOutput = workTable.getOutput(); + for (int i = 0; i < workTableOutput.size() && i < baseOutput.size(); i++) { + derivedSlotInputs.put(workTableOutput.get(i).getExprId(), + ImmutableSet.of(baseOutput.get(i))); + } + } + } + walkPlan(recursiveCase, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); + recordSetOpChildrenOutputs(recursiveUnion.getOutput(), childrenOutputs, + exprIdToScan, exprIdToColName, deltas, derivedSlotInputs, false); + return; + } for (Plan child : plan.children()) { - walkPlan(child, exprIdToScan, exprIdToColName, deltas); + walkPlan(child, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); } if (plan instanceof PhysicalFilter) { PhysicalFilter filter = (PhysicalFilter) plan; for (Expression conjunct : filter.getConjuncts()) { - recordInputSlotsAsFilterHit(conjunct, exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsFilterHit(conjunct, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); } } - // Lazy-materialized columns are not in PhysicalLazyMaterializeOlapScan.getOutput() - // (only operative slots + row-id are). The parent PhysicalLazyMaterialize exposes - // the lazy slots via getLazySlots(). Use each row-id — already registered by the - // child scan branch — to look up the source scan and register the lazy ExprIds. + // Lazy columns are absent from PhysicalLazyMaterializeOlapScan.getOutput(); the parent + // PhysicalLazyMaterialize exposes them via getLazySlots(), linked by row-id to the scan. if (plan instanceof PhysicalLazyMaterialize) { PhysicalLazyMaterialize lazy = (PhysicalLazyMaterialize) plan; List rels = lazy.getRelations(); @@ -254,16 +303,31 @@ private static void walkPlan(Plan plan, Aggregate agg = (Aggregate) plan; // GROUP BY keys for (Expression expr : agg.getGroupByExpressions()) { - recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); } // Columns consumed by aggregate functions (e.g. k2 in SUM(k2)) for (NamedExpression expr : agg.getOutputExpressions()) { - recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); + } + // HAVING: link each non-scan-backed aggregate output to its own inputs, however + // computed those already are (e.g. HAVING SUM(x) where x is itself computed). + for (NamedExpression ne : agg.getOutputExpressions()) { + if (exprIdToScan.containsKey(ne.getExprId()) || ne instanceof Slot) { + // Bare pass-through output (e.g. two-phase DISTINCT's merge phase reusing the + // local phase's slot) — already linked there; getInputSlots() on a bare Slot + // includes itself, which would self-loop here. + continue; + } + Set inputSlots = ne.getInputSlots(); + if (!inputSlots.isEmpty()) { + derivedSlotInputs.put(ne.getExprId(), inputSlots); + } } } if (plan instanceof AbstractPhysicalSort) { for (OrderKey orderKey : ((AbstractPhysicalSort) plan).getOrderKeys()) { - recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan, exprIdToColName, deltas, + derivedSlotInputs); } } // PhysicalPartitionTopN does not extend AbstractPhysicalSort but also has ORDER BY and @@ -271,10 +335,11 @@ private static void walkPlan(Plan plan, if (plan instanceof PhysicalPartitionTopN) { PhysicalPartitionTopN ptn = (PhysicalPartitionTopN) plan; for (Expression partKey : ptn.getPartitionKeys()) { - recordInputSlotsAsQueryHit(partKey, exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsQueryHit(partKey, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); } for (OrderKey orderKey : ptn.getOrderKeys()) { - recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsQueryHit(orderKey.getExpr(), exprIdToScan, exprIdToColName, deltas, + derivedSlotInputs); } } // PhysicalRepeat handles ROLLUP/CUBE: group sets are like GROUP BY keys. @@ -282,44 +347,108 @@ private static void walkPlan(Plan plan, PhysicalRepeat repeat = (PhysicalRepeat) plan; for (List groupSet : repeat.getGroupingSets()) { for (Expression expr : groupSet) { - recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); } } for (NamedExpression expr : repeat.getOutputExpressions()) { - recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); + } + // HAVING GROUPING(a)/GROUPING_ID(...): link each non-scan-backed repeat output + // (e.g. Alias(Grouping(a), G)) to its own inputs, same guard as Aggregate HAVING. + for (NamedExpression ne : repeat.getOutputExpressions()) { + if (exprIdToScan.containsKey(ne.getExprId()) || ne instanceof Slot) { + continue; + } + Set inputSlots = ne.getInputSlots(); + if (!inputSlots.isEmpty()) { + derivedSlotInputs.put(ne.getExprId(), inputSlots); + } } } if (plan instanceof PhysicalWindow) { WindowFrameGroup wfg = ((PhysicalWindow) plan).getWindowFrameGroup(); Set partitionKeys = wfg.getPartitionKeys(); for (Expression expr : partitionKeys) { - recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsQueryHit(expr, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); } for (OrderExpression orderExpr : wfg.getOrderKeys()) { - recordInputSlotsAsQueryHit(orderExpr.child(), exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsQueryHit(orderExpr.child(), exprIdToScan, exprIdToColName, deltas, + derivedSlotInputs); } - // queryHit for the window function value columns (e.g. k2 in SUM(k2) OVER (...)). + // queryHit for window value columns (e.g. k2 in SUM(k2) OVER (...)), and link the + // alias to the window's FULL input set (function args + partition + order keys) for + // a QUALIFY-style filter above. Positional functions like ROW_NUMBER have no function + // arguments, but still have partition/order keys, so windowExpr.getInputSlots() (not + // function.getInputSlots()) is used for the link — WindowExpression wires all three + // as children, so this is a single union, not three separate calls. for (NamedExpression windowAlias : wfg.getGroups()) { Expression windowExpr = windowAlias.child(0); if (windowExpr instanceof WindowExpression) { - recordInputSlotsAsQueryHit( - ((WindowExpression) windowExpr).getFunction(), - exprIdToScan, exprIdToColName, deltas); + Expression function = ((WindowExpression) windowExpr).getFunction(); + recordInputSlotsAsQueryHit(function, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); + if (!exprIdToScan.containsKey(windowAlias.getExprId())) { + Set inputSlots = windowExpr.getInputSlots(); + if (!inputSlots.isEmpty()) { + derivedSlotInputs.put(windowAlias.getExprId(), inputSlots); + } + } } } } - // filterHit for JOIN ON conditions (hash equality and non-equality predicates). + // filterHit for all JOIN ON conditions; mark conjuncts are a separate field not included + // in hashJoinConjuncts or otherJoinConjuncts (IN/EXISTS subquery correlation columns). if (plan instanceof AbstractPhysicalJoin) { AbstractPhysicalJoin join = (AbstractPhysicalJoin) plan; for (Expression conjunct : join.getHashJoinConjuncts()) { - recordInputSlotsAsFilterHit(conjunct, exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsFilterHit(conjunct, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); } for (Expression conjunct : join.getOtherJoinConjuncts()) { - recordInputSlotsAsFilterHit(conjunct, exprIdToScan, exprIdToColName, deltas); + recordInputSlotsAsFilterHit(conjunct, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); + } + for (Expression conjunct : join.getMarkJoinConjuncts()) { + recordInputSlotsAsFilterHit(conjunct, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); + } + } + // UNION / INTERSECT / EXCEPT: queryHit per branch, plus link the set op's own output + // slots to those branch slots so a filter kept above the set op (e.g. a volatile + // predicate PushDownFilterThroughSetOperation can't push down) still resolves. For + // EXCEPT and INTERSECT, only the first (build-side) branch's values ever reach the + // output block at execution time (see SetSourceOperatorX::_add_result_columns / the + // build-side DCHECK(_cur_child_id == 0) in set_sink_operator.cpp) — later branches are + // scanned only to probe/exclude — so the output-slot link is restricted to branch 0. + if (plan instanceof PhysicalSetOperation) { + PhysicalSetOperation setOp = (PhysicalSetOperation) plan; + recordSetOpChildrenOutputs(setOp.getOutput(), setOp.getRegularChildrenOutputs(), + exprIdToScan, exprIdToColName, deltas, derivedSlotInputs, + plan instanceof PhysicalExcept || plan instanceof PhysicalIntersect); + } + // LATERAL VIEW / EXPLODE: queryHit for generator inputs; filterHit for the generator's + // own ON predicate (e.g. table-function join), sent straight to TableFunctionNode. + // generators[i] positionally produces generatorOutput[i] (see MergeGenerates), so link + // each output slot back to its generator's inputs for a parent GROUP BY/filter on it. + if (plan instanceof PhysicalGenerate) { + PhysicalGenerate generate = (PhysicalGenerate) plan; + List generators = generate.getGenerators(); + List generatorOutput = generate.getGeneratorOutput(); + for (int i = 0; i < generators.size(); i++) { + Function generator = generators.get(i); + recordInputSlotsAsQueryHit(generator, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); + if (i >= generatorOutput.size()) { + continue; + } + Slot outputSlot = generatorOutput.get(i); + if (!exprIdToScan.containsKey(outputSlot.getExprId())) { + Set inputSlots = generator.getInputSlots(); + if (!inputSlots.isEmpty()) { + derivedSlotInputs.put(outputSlot.getExprId(), inputSlots); + } + } + } + for (Expression conjunct : generate.getConjuncts()) { + recordInputSlotsAsFilterHit(conjunct, exprIdToScan, exprIdToColName, deltas, derivedSlotInputs); } } - // Propagate alias ExprIds for intermediate PhysicalProject nodes so that parent - // plan output slots (derived from aliases) resolve back to the original scan. + // Link alias ExprIds to their input slots so parent output resolves back to the scan. if (plan instanceof PhysicalProject) { for (NamedExpression ne : ((PhysicalProject) plan).getProjects()) { if (exprIdToScan.containsKey(ne.getExprId())) { @@ -327,56 +456,137 @@ private static void walkPlan(Plan plan, } SlotReference underlying = unwrapAlias(ne); if (underlying != null && !underlying.getExprId().equals(ne.getExprId())) { - // Simple alias: Alias(SlotRef) — propagate scan and column name. - PhysicalOlapScan scan = exprIdToScan.get(underlying.getExprId()); - if (scan != null) { - exprIdToScan.put(ne.getExprId(), scan); - String colName = exprIdToColName.get(underlying.getExprId()); - if (colName != null) { - exprIdToColName.put(ne.getExprId(), colName); - } - } + // Simple alias: Alias(SlotRef) — one input slot, same identity chain. + derivedSlotInputs.put(ne.getExprId(), ImmutableSet.of(underlying)); } else if (underlying == null && ne instanceof Alias) { - // Complex alias: Alias(Cast(col), name) — created by - // PushDownExpressionsInHashCondition for type-mismatched join keys. - // If all input slots trace back to one scan, propagate to the alias ExprId. + // Computed alias: Alias(k1+k2), Alias(Cast(k1)) join-key cast, etc. Set inputSlots = ((Alias) ne).child().getInputSlots(); - if (inputSlots.size() == 1) { - Slot inputSlot = inputSlots.iterator().next(); - PhysicalOlapScan scan = exprIdToScan.get(inputSlot.getExprId()); - if (scan != null) { - exprIdToScan.put(ne.getExprId(), scan); - String colName = exprIdToColName.get(inputSlot.getExprId()); - if (colName != null) { - exprIdToColName.put(ne.getExprId(), colName); - } - } + if (!inputSlots.isEmpty()) { + derivedSlotInputs.put(ne.getExprId(), inputSlots); } } } } } - private static void recordInputSlotsAsQueryHit(Expression expr, + /** + * Resolves an ExprId down to the base scan-backed ExprIds it derives from, recursing + * through derivedSlotInputs for any non-scan slot. Shared by every call site. + */ + private static Set resolveBaseScanExprIds(ExprId exprId, + Map exprIdToScan, + Map> derivedSlotInputs) { + return resolveBaseScanExprIds(exprId, exprIdToScan, derivedSlotInputs, new HashSet<>()); + } + + // visiting guards against a cyclic derivedSlotInputs entry causing unbounded recursion. + private static Set resolveBaseScanExprIds(ExprId exprId, + Map exprIdToScan, + Map> derivedSlotInputs, + Set visiting) { + if (exprIdToScan.containsKey(exprId)) { + return ImmutableSet.of(exprId); + } + if (!visiting.add(exprId)) { + return ImmutableSet.of(); + } + Set inputSlots = derivedSlotInputs.get(exprId); + if (inputSlots == null) { + return ImmutableSet.of(); + } + ImmutableSet.Builder result = ImmutableSet.builder(); + for (Slot slot : inputSlots) { + if (slot instanceof SlotReference) { + result.addAll(resolveBaseScanExprIds(slot.getExprId(), exprIdToScan, derivedSlotInputs, visiting)); + } + } + return result.build(); + } + + private static void recordExprIdAsQueryHit(ExprId baseExprId, + Map exprIdToScan, + Map exprIdToColName, + Map deltas) { + PhysicalOlapScan scan = exprIdToScan.get(baseExprId); + if (scan == null) { + return; + } + StatsDelta delta = getOrCreateDelta(deltas, scan); + if (delta == null) { + return; + } + String colName = exprIdToColName.get(baseExprId); + if (colName != null) { + delta.addQueryStats(colName); + } + } + + private static void recordExprIdAsFilterHit(ExprId baseExprId, Map exprIdToScan, Map exprIdToColName, Map deltas) { + PhysicalOlapScan scan = exprIdToScan.get(baseExprId); + if (scan == null) { + return; + } + StatsDelta delta = getOrCreateDelta(deltas, scan); + if (delta == null) { + return; + } + String colName = exprIdToColName.get(baseExprId); + if (colName != null) { + delta.addFilterStats(colName); + } + } + + /** Shared by PhysicalSetOperation and PhysicalRecursiveUnion — both expose the same + * getOutput()/getRegularChildrenOutputs() contract and need identical recording logic. + * onlyFirstBranchLinksOutput is true for EXCEPT: every branch is still scanned and gets + * queryHit, but only the first branch's values ever reach the output, so a filter kept + * above the EXCEPT must not resolve back to a later (subtracted-away) branch's column. */ + private static void recordSetOpChildrenOutputs( + List setOpOutput, + List> childrenOutputs, + Map exprIdToScan, + Map exprIdToColName, + Map deltas, + Map> derivedSlotInputs, + boolean onlyFirstBranchLinksOutput) { + for (int branchIdx = 0; branchIdx < childrenOutputs.size(); branchIdx++) { + List childOutput = childrenOutputs.get(branchIdx); + boolean linkOutput = !onlyFirstBranchLinksOutput || branchIdx == 0; + for (int i = 0; i < childOutput.size(); i++) { + SlotReference branchSlot = childOutput.get(i); + for (ExprId baseId : resolveBaseScanExprIds(branchSlot.getExprId(), exprIdToScan, + derivedSlotInputs)) { + recordExprIdAsQueryHit(baseId, exprIdToScan, exprIdToColName, deltas); + } + if (!linkOutput || i >= setOpOutput.size()) { + continue; + } + Slot outputSlot = setOpOutput.get(i); + if (outputSlot instanceof SlotReference && !exprIdToScan.containsKey(outputSlot.getExprId()) + && !outputSlot.getExprId().equals(branchSlot.getExprId())) { + // Skip self-reuse (output slot IS the branch slot) — would self-loop; the + // branch's queryHit is already recorded above regardless. + derivedSlotInputs.computeIfAbsent(outputSlot.getExprId(), k -> new HashSet<>()) + .add(branchSlot); + } + } + } + } + + private static void recordInputSlotsAsQueryHit(Expression expr, + Map exprIdToScan, + Map exprIdToColName, + Map deltas, + Map> derivedSlotInputs) { for (Slot slot : expr.getInputSlots()) { if (!(slot instanceof SlotReference)) { continue; } - SlotReference sr = (SlotReference) slot; - PhysicalOlapScan sourceScan = exprIdToScan.get(sr.getExprId()); - if (sourceScan == null) { - continue; - } - StatsDelta delta = getOrCreateDelta(deltas, sourceScan); - if (delta != null) { - String colName = sr.getOriginalColumn().map(col -> col.getName()) - .orElseGet(() -> exprIdToColName.get(sr.getExprId())); - if (colName != null) { - delta.addQueryStats(colName); - } + for (ExprId baseId : resolveBaseScanExprIds(slot.getExprId(), exprIdToScan, derivedSlotInputs)) { + recordExprIdAsQueryHit(baseId, exprIdToScan, exprIdToColName, deltas); } } } @@ -384,23 +594,14 @@ private static void recordInputSlotsAsQueryHit(Expression expr, private static void recordInputSlotsAsFilterHit(Expression expr, Map exprIdToScan, Map exprIdToColName, - Map deltas) { + Map deltas, + Map> derivedSlotInputs) { for (Slot slot : expr.getInputSlots()) { if (!(slot instanceof SlotReference)) { continue; } - SlotReference sr = (SlotReference) slot; - PhysicalOlapScan sourceScan = exprIdToScan.get(sr.getExprId()); - if (sourceScan == null) { - continue; - } - StatsDelta delta = getOrCreateDelta(deltas, sourceScan); - if (delta != null) { - String colName = sr.getOriginalColumn().map(col -> col.getName()) - .orElseGet(() -> exprIdToColName.get(sr.getExprId())); - if (colName != null) { - delta.addFilterStats(colName); - } + for (ExprId baseId : resolveBaseScanExprIds(slot.getExprId(), exprIdToScan, derivedSlotInputs)) { + recordExprIdAsFilterHit(baseId, exprIdToScan, exprIdToColName, deltas); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 10b139b73f047f..d683824699917e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -605,12 +605,22 @@ public static Optional getIcebergColumnStats(String colName, or try (CloseableIterable fileScanTasks = tableScan.planFiles()) { for (FileScanTask task : fileScanTasks) { int colId = getColId(task.spec(), colName); - totalDataSize += task.file().columnSizes().get(colId); + Map columnSizes = task.file().columnSizes(); + Map nullValueCounts = task.file().nullValueCounts(); + Long columnSize = columnSizes == null ? null : columnSizes.get(colId); + Long nullValueCount = nullValueCounts == null ? null : nullValueCounts.get(colId); + // Iceberg can omit maps or entries for mode=none; partial aggregation would fabricate zero stats. + if (columnSize == null || nullValueCount == null) { + return Optional.empty(); + } + totalDataSize += columnSize; totalDataCount += task.file().recordCount(); - totalNumNull += task.file().nullValueCounts().get(colId); + totalNumNull += nullValueCount; } } catch (IOException e) { LOG.warn("Error to close FileScanTask.", e); + // A failed close can cancel an in-flight empty return, so accumulated stats are not reliable. + return Optional.empty(); } ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(totalDataCount); columnStatisticBuilder.setMaxValue(Double.POSITIVE_INFINITY); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Frontend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Frontend.java index 2de174c86ef7bb..62970a9315cc6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/Frontend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/Frontend.java @@ -50,6 +50,7 @@ public class Frontend implements Writable { private String cloudUniqueId; private String version; + private transient String localResourceGroup = ""; private int queryPort; private int rpcPort; @@ -92,6 +93,10 @@ public String getVersion() { return version; } + public String getLocalResourceGroup() { + return localResourceGroup; + } + public String getNodeName() { return nodeName; } @@ -152,6 +157,10 @@ public void setCloudUniqueId(String cloudUniqueId) { this.cloudUniqueId = cloudUniqueId; } + public void setLocalResourceGroup(String localResourceGroup) { + this.localResourceGroup = localResourceGroup; + } + public String getCloudUniqueId() { return cloudUniqueId; } @@ -183,6 +192,9 @@ public boolean handleHbResponse(FrontendHbResponse hbResponse, boolean isReplay) heartbeatErrMsg = ""; lastStartupTime = hbResponse.getFeStartTime(); diskInfos = hbResponse.getDiskInfos(); + if (hbResponse.getLocalResourceGroup() != null) { + setLocalResourceGroup(hbResponse.getLocalResourceGroup()); + } isChanged = true; processUUID = hbResponse.getProcessUUID(); } else { @@ -196,6 +208,7 @@ public boolean handleHbResponse(FrontendHbResponse hbResponse, boolean isReplay) isChanged = true; } heartbeatErrMsg = hbResponse.getMsg() == null ? "Unknown error" : hbResponse.getMsg(); + setLocalResourceGroup(""); } return isChanged; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/FrontendHbResponse.java b/fe/fe-core/src/main/java/org/apache/doris/system/FrontendHbResponse.java index 0bbc72008dd022..3a16cfd5a97edb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/FrontendHbResponse.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/FrontendHbResponse.java @@ -44,6 +44,7 @@ public class FrontendHbResponse extends HeartbeatResponse implements Writable { private long feStartTime; private long processUUID; private List diskInfos; + private String localResourceGroup; public FrontendHbResponse() { super(HeartbeatResponse.Type.FRONTEND); @@ -52,7 +53,7 @@ public FrontendHbResponse() { public FrontendHbResponse(String name, int queryPort, int rpcPort, int arrowFlightSqlPort, long replayedJournalId, long hbTime, String version, long feStartTime, List diskInfos, - long processUUID) { + long processUUID, String localResourceGroup) { super(HeartbeatResponse.Type.FRONTEND); this.status = HbStatus.OK; this.name = name; @@ -65,6 +66,7 @@ public FrontendHbResponse(String name, int queryPort, int rpcPort, int arrowFlig this.feStartTime = feStartTime; this.diskInfos = diskInfos; this.processUUID = processUUID; + this.localResourceGroup = localResourceGroup; } public FrontendHbResponse(String name, String errMsg) { @@ -111,6 +113,10 @@ public List getDiskInfos() { return diskInfos; } + public String getLocalResourceGroup() { + return localResourceGroup; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -121,7 +127,9 @@ public String toString() { sb.append(", rpcPort: ").append(rpcPort); sb.append(", arrowFlightSqlPort: ").append(arrowFlightSqlPort); sb.append(", replayedJournalId: ").append(replayedJournalId); - sb.append(", festartTime: ").append(processUUID); + sb.append(", feStartTime: ").append(feStartTime); + sb.append(", processUUID: ").append(processUUID); + sb.append(", localResourceGroup: ").append(localResourceGroup); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java index f23af939399a9d..dbd36715d53641 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java @@ -419,7 +419,7 @@ public HeartbeatResponse call() { System.currentTimeMillis(), Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH, ExecuteEnv.getInstance().getStartupTime(), ExecuteEnv.getInstance().getDiskInfos(), - ExecuteEnv.getInstance().getProcessUUID()); + ExecuteEnv.getInstance().getProcessUUID(), Config.local_resource_group); } else { return new FrontendHbResponse(fe.getNodeName(), "not ready"); } @@ -442,7 +442,8 @@ private HeartbeatResponse getHeartbeatResponse() { return new FrontendHbResponse(fe.getNodeName(), result.getQueryPort(), result.getRpcPort(), result.getArrowFlightSqlPort(), result.getReplayedJournalId(), System.currentTimeMillis(), result.getVersion(), result.getLastStartupTime(), - FeDiskInfo.fromThrifts(result.getDiskInfos()), result.getProcessUUID()); + FeDiskInfo.fromThrifts(result.getDiskInfos()), result.getProcessUUID(), + result.getLocalResourceGroup()); } else { return new FrontendHbResponse(fe.getNodeName(), result.getMsg()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CdcStreamTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CdcStreamTableValuedFunction.java index 6dec7cd6bef945..e4323cef82ab15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CdcStreamTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CdcStreamTableValuedFunction.java @@ -27,6 +27,7 @@ import org.apache.doris.job.cdc.request.FetchRecordRequest; import org.apache.doris.job.common.DataSourceType; import org.apache.doris.job.extensions.insert.streaming.DataSourceConfigValidator; +import org.apache.doris.job.extensions.insert.streaming.StreamingJdbcUrlNormalizer; import org.apache.doris.job.util.StreamingJobUtils; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileType; @@ -68,14 +69,17 @@ private void processProps(Map properties) throws AnalysisExcepti copyProps.remove(INCLUDE_DELETE_SIGN); copyProps.put("format", "json"); + DataSourceType sourceType = DataSourceType.valueOf( + copyProps.get(DataSourceConfigKeys.TYPE).toUpperCase()); + copyProps.put(DataSourceConfigKeys.JDBC_URL, StreamingJdbcUrlNormalizer.normalize( + sourceType, copyProps.get(DataSourceConfigKeys.JDBC_URL))); + // Standalone TVF: random jobId. TVF-in-job: job.id injected by rewriteTvfParams. String jobId = copyProps.computeIfAbsent(JOB_ID_KEY, k -> UUID.randomUUID().toString().replace("-", "")); // Default PG slot/pub so cdcclient auto-creates per-job resources - StreamingJobUtils.populateDefaultSourceProperties( - DataSourceType.valueOf(copyProps.get(DataSourceConfigKeys.TYPE).toUpperCase()), - copyProps, jobId); + StreamingJobUtils.populateDefaultSourceProperties(sourceType, copyProps, jobId); super.parseCommonProperties(copyProps); this.processedParams.put(ENABLE_CDC_CLIENT_KEY, "true"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java index 05b709cd523e3b..9df36a534a5ffd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java @@ -64,7 +64,8 @@ public class FrontendsTableValuedFunction extends MetadataTableValuedFunction { new Column("ErrMsg", ScalarType.createStringType()), new Column("Version", ScalarType.createStringType()), new Column("CurrentConnected", ScalarType.createStringType()), - new Column("LiveSince", ScalarType.createStringType()) + new Column("LiveSince", ScalarType.createStringType()), + new Column("LocalResourceGroup", ScalarType.createStringType()) ); private static final ImmutableMap COLUMN_TO_INDEX; private static final ImmutableList TITLE_NAMES; diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java index 122014f0e8b2c2..3ad88c49c408bd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java @@ -28,6 +28,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.plugin.audit.AuditLoader; import org.apache.doris.statistics.StatisticConstants; import org.apache.doris.utframe.TestWithFeService; @@ -67,6 +68,10 @@ private void checkReplicationNum(Database db, String tblName) throws AnalysisExc PartitionInfo partitionInfo = olapTable.getPartitionInfo(); Assertions.assertEquals((short) 3, olapTable.getTableProperty().getReplicaAllocation().getTotalReplicaNum(), tblName); + Assertions.assertFalse(olapTable.getTableProperty().getProperties() + .containsKey("default." + PropertyAnalyzer.PROPERTIES_REPLICATION_NUM), tblName); + Assertions.assertTrue(olapTable.getTableProperty().getProperties() + .containsKey("default." + PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION), tblName); for (Partition partition : olapTable.getPartitions()) { Assertions.assertEquals((short) 3, partitionInfo.getReplicaAllocation(partition.getId()).getTotalReplicaNum()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index 040b43aaf0a4ba..7c910231f6a16c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.info.ColumnPosition; import org.apache.doris.catalog.info.IndexType; @@ -37,6 +38,7 @@ import org.apache.doris.qe.StmtExecutor; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; @@ -46,8 +48,11 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.lang.reflect.Method; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.IntSupplier; import java.util.stream.Collectors; public class SchemaChangeHandlerTest extends TestWithFeService { @@ -888,6 +893,43 @@ public void testAddValueColumnOnAggMV() { } + @Test + public void testRowBinlogSchemaChangeKeepsHiddenKeyColumn() throws Exception { + SchemaChangeHandler schemaChangeHandler = new SchemaChangeHandler(); + List rowBinlogSchema = Lists.newArrayList(); + Column key = new Column("k1", PrimitiveType.INT); + key.setIsKey(true); + rowBinlogSchema.add(Column.generateRowBinlogKeyColumn(key)); + rowBinlogSchema.add(new Column(Column.BINLOG_TSO_COL, PrimitiveType.BIGINT)); + rowBinlogSchema.add(new Column(Column.BINLOG_LSN_COL, PrimitiveType.BIGINT)); + rowBinlogSchema.add(new Column(Column.BINLOG_OPERATION_COL, PrimitiveType.BIGINT)); + + Column hiddenKey = new Column("__DORIS_TEST_HIDDEN_KEY__", PrimitiveType.BIGINT); + hiddenKey.setIsKey(true); + hiddenKey.setIsVisible(false); + AtomicInteger uniqueId = new AtomicInteger(10); + IntSupplier uniqueIdSupplier = uniqueId::getAndIncrement; + Method addColumnRowBinlog = SchemaChangeHandler.class.getDeclaredMethod( + "addColumnRowBinlog", List.class, Column.class, ColumnPosition.class, + java.util.Set.class, boolean.class, IntSupplier.class); + addColumnRowBinlog.setAccessible(true); + addColumnRowBinlog.invoke(schemaChangeHandler, rowBinlogSchema, hiddenKey, null, + Sets.newHashSet(hiddenKey.getName()), false, uniqueIdSupplier); + + List columnNames = rowBinlogSchema.stream().map(Column::getName).collect(Collectors.toList()); + Assertions.assertEquals(1, columnNames.indexOf("__DORIS_TEST_HIDDEN_KEY__")); + Assertions.assertEquals(2, columnNames.indexOf(Column.BINLOG_TSO_COL)); + Assertions.assertEquals(3, columnNames.indexOf(Column.BINLOG_LSN_COL)); + + Column hiddenValue = new Column("__DORIS_TEST_HIDDEN_VALUE__", PrimitiveType.INT); + hiddenValue.setIsKey(false); + hiddenValue.setIsVisible(false); + addColumnRowBinlog.invoke(schemaChangeHandler, rowBinlogSchema, hiddenValue, null, + Sets.newHashSet(hiddenValue.getName()), false, uniqueIdSupplier); + columnNames = rowBinlogSchema.stream().map(Column::getName).collect(Collectors.toList()); + Assertions.assertFalse(columnNames.contains("__DORIS_TEST_HIDDEN_VALUE__")); + } + @Test public void testAggAddOrDropInvertedIndex() throws Exception { LOG.info("dbName: {}", Env.getCurrentInternalCatalog().getDbNames()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToThriftBehaviorTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToThriftBehaviorTest.java index 68e69cf3e27a0d..ac40a4f174f682 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToThriftBehaviorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToThriftBehaviorTest.java @@ -274,6 +274,22 @@ public void testTreesToThrift() { Assertions.assertEquals(TExprNodeType.STRING_LITERAL, result.get(2).getNodes().get(0).node_type); } + @Test + public void testLambdaFunctionExprSerializesArgumentNames() { + SlotRef slotX = new SlotRef(null, "x"); + SlotRef slotY = new SlotRef(null, "yy"); + LambdaFunctionExpr expr = new LambdaFunctionExpr( + new IntLiteral(1L), Lists.newArrayList("x", "yy"), + Lists.newArrayList(slotX, slotY), false); + + TExprNode node = firstNode(expr); + + Assertions.assertEquals(TExprNodeType.LAMBDA_FUNCTION_EXPR, node.node_type); + Assertions.assertFalse(node.isSetLabel()); + Assertions.assertTrue(node.isSetLambdaArgumentNames()); + Assertions.assertEquals(Lists.newArrayList("x", "yy"), node.getLambdaArgumentNames()); + } + // ======================== Helpers ======================== private static TExprNode firstNode(Expr expr) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableRowBinlogSchemaTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableRowBinlogSchemaTest.java index 42a3bc8e078ccb..4e788c24b0e70c 100755 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableRowBinlogSchemaTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableRowBinlogSchemaTest.java @@ -30,12 +30,15 @@ public class OlapTableRowBinlogSchemaTest { private static OlapTable newTestTable(BinlogConfig binlogConfig) { - long baseIndexId = 1L; Column key = new Column("k1", PrimitiveType.INT); key.setIsKey(true); Column value = new Column("v1", Type.INT, false, null, false, "7", ""); value.setIsKey(false); - List baseSchema = Lists.newArrayList(key, value); + return newTestTable(binlogConfig, Lists.newArrayList(key, value)); + } + + private static OlapTable newTestTable(BinlogConfig binlogConfig, List baseSchema) { + long baseIndexId = 1L; // Construct a minimal olap table for row binlog schema generation. OlapTable table = new OlapTable(1L, "tbl", baseSchema, KeysType.PRIMARY_KEYS, null, null); @@ -113,4 +116,53 @@ public void testRowBinlogSchemaOnDisable() { Assertions.assertFalse(table.needRowBinlog()); Assertions.assertTrue(table.getBaseIndexMeta().getRowBinlogIndexId() <= 0); } + + @Test + public void testRowBinlogSchemaIncludesHiddenKeyColumns() { + Column key = new Column("k1", PrimitiveType.INT); + key.setIsKey(true); + Column hiddenKey = new Column("__DORIS_TEST_HIDDEN_KEY__", PrimitiveType.BIGINT); + hiddenKey.setIsKey(true); + hiddenKey.setIsVisible(false); + Column value = new Column("v1", PrimitiveType.INT); + value.setIsKey(false); + Column hiddenValue = new Column("__DORIS_TEST_HIDDEN_VALUE__", PrimitiveType.INT); + hiddenValue.setIsKey(false); + hiddenValue.setIsVisible(false); + + OlapTable table = newTestTable(BinlogTestUtils.newTestRowBinlogConfig(true, true), + Lists.newArrayList(key, hiddenKey, value, hiddenValue)); + + List columnNames = table.getRowBinlogMeta().getSchema(true).stream().map(Column::getName) + .collect(Collectors.toList()); + Assertions.assertEquals("k1", columnNames.get(0)); + Assertions.assertEquals("__DORIS_TEST_HIDDEN_KEY__", columnNames.get(1)); + Assertions.assertEquals("v1", columnNames.get(2)); + Assertions.assertFalse(columnNames.contains("__DORIS_TEST_HIDDEN_VALUE__")); + Assertions.assertTrue(columnNames.contains(Column.generateBeforeColName("v1"))); + Assertions.assertFalse(columnNames.contains(Column.generateBeforeColName("__DORIS_TEST_HIDDEN_KEY__"))); + Assertions.assertEquals(4, columnNames.indexOf(Column.BINLOG_TSO_COL)); + Assertions.assertEquals(5, columnNames.indexOf(Column.BINLOG_LSN_COL)); + } + + @Test + public void testRowBinlogSchemaSkipsHiddenNonKeyColumnBeforeVisibleColumn() { + Column key = new Column("k1", PrimitiveType.INT); + key.setIsKey(true); + Column hiddenValue = new Column("__DORIS_TEST_HIDDEN_VALUE__", PrimitiveType.INT); + hiddenValue.setIsKey(false); + hiddenValue.setIsVisible(false); + Column value = new Column("v1", PrimitiveType.INT); + value.setIsKey(false); + + OlapTable table = newTestTable(BinlogTestUtils.newTestRowBinlogConfig(true, false), + Lists.newArrayList(key, hiddenValue, value)); + List columnNames = table.getRowBinlogMeta().getSchema(true).stream().map(Column::getName) + .collect(Collectors.toList()); + Assertions.assertEquals("k1", columnNames.get(0)); + Assertions.assertEquals("v1", columnNames.get(1)); + Assertions.assertFalse(columnNames.contains("__DORIS_TEST_HIDDEN_VALUE__")); + Assertions.assertEquals(2, columnNames.indexOf(Column.BINLOG_TSO_COL)); + Assertions.assertEquals(3, columnNames.indexOf(Column.BINLOG_LSN_COL)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TablePropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TablePropertyTest.java index 7cb8d46b24ca07..dd250e44e9f9c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TablePropertyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TablePropertyTest.java @@ -17,13 +17,23 @@ package org.apache.doris.catalog; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.resource.Tag; + import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.util.Map; public class TablePropertyTest { + private static final String DEFAULT_REPLICATION_NUM = + "default." + PropertyAnalyzer.PROPERTIES_REPLICATION_NUM; + private static final String DEFAULT_REPLICATION_ALLOCATION = + "default." + PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION; + private static final String REPLICATION_ALLOCATION = + "tag.location.group_0: 1, tag.location.group_1: 1, tag.location.group_2: 1"; // A non-whitelisted dynamic_partition.* key is ignored (skipped via continue), so it is not // collected at all and the table is neither built as dynamic nor flagged as incomplete. @@ -105,4 +115,80 @@ public void testCompleteDynamicPartitionIsBuilt() { Assert.assertEquals(3, tableProperty.getDynamicPartitionProperty().getEnd()); Assert.assertEquals(1, tableProperty.getDynamicPartitionProperty().getBuckets()); } + + @Test + public void testModifyDefaultReplicaAllocationRemovesLegacyReplicationNum() { + Map properties = Maps.newHashMap(); + properties.put(DEFAULT_REPLICATION_NUM, "3"); + TableProperty tableProperty = new TableProperty(properties); + tableProperty.buildReplicaAllocation(); + + Map modifiedProperties = Maps.newHashMap(); + modifiedProperties.put(DEFAULT_REPLICATION_ALLOCATION, REPLICATION_ALLOCATION); + tableProperty.modifyTableProperties(modifiedProperties); + tableProperty.buildReplicaAllocation(); + + assertReplicaAllocationWins(tableProperty); + } + + @Test + public void testDeserializeConflictingDefaultReplicaPropertiesPreservesNumericPrecedence() throws IOException { + Map properties = Maps.newHashMap(); + properties.put(DEFAULT_REPLICATION_NUM, "3"); + properties.put(DEFAULT_REPLICATION_ALLOCATION, REPLICATION_ALLOCATION); + TableProperty tableProperty = new TableProperty(properties); + + tableProperty.gsonPostProcess(); + + Assert.assertTrue(tableProperty.getProperties().containsKey(DEFAULT_REPLICATION_NUM)); + Assert.assertTrue(tableProperty.getProperties().containsKey(DEFAULT_REPLICATION_ALLOCATION)); + Assert.assertEquals(Short.valueOf((short) 3), + tableProperty.getReplicaAllocation().getReplicaNumByTag(Tag.DEFAULT_BACKEND_TAG)); + } + + @Test + public void testResetPropertiesForRestoreRemovesLegacyReplicationNum() { + Map properties = Maps.newHashMap(); + properties.put(DEFAULT_REPLICATION_NUM, "3"); + TableProperty tableProperty = new TableProperty(properties); + tableProperty.buildReplicaAllocation(); + + ReplicaAllocation restoredReplicaAllocation = new ReplicaAllocation((short) 2); + tableProperty.resetPropertiesForRestore(false, false, restoredReplicaAllocation); + + Assert.assertFalse(tableProperty.getProperties().containsKey(DEFAULT_REPLICATION_NUM)); + Assert.assertEquals(restoredReplicaAllocation.toCreateStmt(), + tableProperty.getProperties().get(DEFAULT_REPLICATION_ALLOCATION)); + Assert.assertEquals((short) 2, tableProperty.getReplicaAllocation().getTotalReplicaNum()); + } + + @Test + public void testModifyDefaultReplicationNumRemovesExistingAllocation() { + Map properties = Maps.newHashMap(); + properties.put(DEFAULT_REPLICATION_ALLOCATION, REPLICATION_ALLOCATION); + TableProperty tableProperty = new TableProperty(properties); + tableProperty.buildReplicaAllocation(); + + Map modifiedProperties = Maps.newHashMap(); + modifiedProperties.put(DEFAULT_REPLICATION_NUM, "2"); + tableProperty.modifyTableProperties(modifiedProperties); + tableProperty.buildReplicaAllocation(); + + Assert.assertFalse(tableProperty.getProperties().containsKey(DEFAULT_REPLICATION_ALLOCATION)); + Assert.assertEquals(Short.valueOf((short) 2), + tableProperty.getReplicaAllocation().getReplicaNumByTag(Tag.DEFAULT_BACKEND_TAG)); + } + + private void assertReplicaAllocationWins(TableProperty tableProperty) { + Assert.assertFalse(tableProperty.getProperties().containsKey(DEFAULT_REPLICATION_NUM)); + Assert.assertEquals(Short.valueOf((short) 1), + tableProperty.getReplicaAllocation() + .getReplicaNumByTag(Tag.createNotCheck(Tag.TYPE_LOCATION, "group_0"))); + Assert.assertEquals(Short.valueOf((short) 1), + tableProperty.getReplicaAllocation() + .getReplicaNumByTag(Tag.createNotCheck(Tag.TYPE_LOCATION, "group_1"))); + Assert.assertEquals(Short.valueOf((short) 1), + tableProperty.getReplicaAllocation() + .getReplicaNumByTag(Tag.createNotCheck(Tag.TYPE_LOCATION, "group_2"))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/stream/TableStreamBaseTableInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/stream/TableStreamBaseTableInfoTest.java new file mode 100644 index 00000000000000..366002384140dd --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/stream/TableStreamBaseTableInfoTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog.stream; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.common.collect.ImmutableList; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TableStreamBaseTableInfoTest { + + @Test + public void testConstructionAndSerialization() { + TableIf table = Mockito.mock(TableIf.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(table.getDatabase()).thenReturn(database); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(table.getId()).thenReturn(3L); + Mockito.when(database.getId()).thenReturn(2L); + Mockito.when(catalog.getId()).thenReturn(1L); + Mockito.when(table.getName()).thenReturn("table_name"); + Mockito.when(database.getFullName()).thenReturn("db_name"); + Mockito.when(catalog.getName()).thenReturn("catalog_name"); + + TableStreamBaseTableInfo baseTableInfo = new TableStreamBaseTableInfo(table); + Assertions.assertEquals(3L, baseTableInfo.getTableId()); + Assertions.assertEquals(2L, baseTableInfo.getDbId()); + Assertions.assertEquals(1L, baseTableInfo.getCtlId()); + Assertions.assertEquals("table_name", baseTableInfo.getTableName()); + Assertions.assertEquals("db_name", baseTableInfo.getDbName()); + Assertions.assertEquals("catalog_name", baseTableInfo.getCtlName()); + Assertions.assertEquals(ImmutableList.of("catalog_name", "db_name", "table_name"), + baseTableInfo.getFullQualifiers()); + + OlapTableStream tableStream = new OlapTableStream(); + tableStream.baseTableInfo = baseTableInfo; + JsonObject json = JsonParser.parseString(GsonUtils.GSON.toJson(tableStream)).getAsJsonObject(); + Assertions.assertTrue(json.has("bti")); + Assertions.assertFalse(json.has("tsi")); + + OlapTableStream deserialized = GsonUtils.GSON.fromJson(json, OlapTableStream.class); + Assertions.assertEquals(baseTableInfo, deserialized.baseTableInfo); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/WarmUpClusterOnTablesParseTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/WarmUpClusterOnTablesParseTest.java index 8bec1d2d5eb394..ceee9533386d4a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/WarmUpClusterOnTablesParseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/WarmUpClusterOnTablesParseTest.java @@ -20,7 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.cloud.OnTablesFilter.TableFilterRule; import org.apache.doris.cloud.OnTablesFilter.TableFilterRule.RuleType; -import org.apache.doris.cloud.catalog.ComputeGroup; +import org.apache.doris.cloud.catalog.CloudComputeGroupMeta; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -103,14 +103,14 @@ private CloudSystemInfoService buildCloudSystemInfoWithVirtualComputeGroup( private void addVirtualComputeGroup(CloudSystemInfoService cloudSys, String virtualComputeGroupName, String activeComputeGroupName, String standbyComputeGroupName) { - ComputeGroup activeComputeGroup = new ComputeGroup(activeComputeGroupName + "_id", - activeComputeGroupName, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup standbyComputeGroup = new ComputeGroup(standbyComputeGroupName + "_id", - standbyComputeGroupName, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup virtualComputeGroup = new ComputeGroup(virtualComputeGroupName + "_id", - virtualComputeGroupName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta activeComputeGroup = new CloudComputeGroupMeta(activeComputeGroupName + "_id", + activeComputeGroupName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta standbyComputeGroup = new CloudComputeGroupMeta(standbyComputeGroupName + "_id", + standbyComputeGroupName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta virtualComputeGroup = new CloudComputeGroupMeta(virtualComputeGroupName + "_id", + virtualComputeGroupName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); virtualComputeGroup.setSubComputeGroups(Arrays.asList(activeComputeGroupName, standbyComputeGroupName)); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(activeComputeGroupName); policy.setStandbyComputeGroup(standbyComputeGroupName); virtualComputeGroup.setPolicy(policy); @@ -370,7 +370,7 @@ public void testOnTablesLoadEventValidateAllowsDestinationComputeGroupOwnedByVir CloudSystemInfoService cloudSys = buildCloudSystemInfoWithVirtualComputeGroup( "vcg", "active_cg", "standby_cg"); cloudSys.addComputeGroup("outside_cg_id", - new ComputeGroup("outside_cg_id", "outside_cg", ComputeGroup.ComputeTypeEnum.COMPUTE)); + new CloudComputeGroupMeta("outside_cg_id", "outside_cg", CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE)); setField(env, Env.class, "systemInfo", cloudSys); WarmUpClusterCommand cmd = parse( "WARM UP CLUSTER standby_cg WITH CLUSTER outside_cg " @@ -398,7 +398,7 @@ public void testOnTablesLoadEventValidateAllowsSourceComputeGroupOwnedByVirtualC CloudSystemInfoService cloudSys = buildCloudSystemInfoWithVirtualComputeGroup( "vcg", "active_cg", "standby_cg"); cloudSys.addComputeGroup("outside_cg_id", - new ComputeGroup("outside_cg_id", "outside_cg", ComputeGroup.ComputeTypeEnum.COMPUTE)); + new CloudComputeGroupMeta("outside_cg_id", "outside_cg", CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE)); setField(env, Env.class, "systemInfo", cloudSys); WarmUpClusterCommand cmd = parse( "WARM UP CLUSTER outside_cg WITH CLUSTER active_cg " diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/ComputeGroupTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudComputeGroupMetaTest.java similarity index 61% rename from fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/ComputeGroupTest.java rename to fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudComputeGroupMetaTest.java index c61f2abefa42fc..2e4d84a4b7f0e2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/ComputeGroupTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudComputeGroupMetaTest.java @@ -26,12 +26,13 @@ import java.util.Map; -public class ComputeGroupTest { - private ComputeGroup computeGroup; +public class CloudComputeGroupMetaTest { + private CloudComputeGroupMeta computeGroup; @BeforeEach public void setUp() { - computeGroup = new ComputeGroup("test_id", "test_group", ComputeGroup.ComputeTypeEnum.COMPUTE); + computeGroup = new CloudComputeGroupMeta("test_id", "test_group", + CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); } @Test @@ -44,16 +45,16 @@ public void testCheckPropertiesWithNull() throws DdlException { public void testCheckPropertiesWithValidBalanceType() throws DdlException { // 测试有效的balance_type Map properties = Maps.newHashMap(); - properties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + properties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); computeGroup.checkProperties(properties); - properties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + properties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); computeGroup.checkProperties(properties); - properties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); + properties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); computeGroup.checkProperties(properties); - properties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.PEER_READ_ASYNC_WARMUP.getValue()); + properties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.PEER_READ_ASYNC_WARMUP.getValue()); computeGroup.checkProperties(properties); } @@ -61,7 +62,7 @@ public void testCheckPropertiesWithValidBalanceType() throws DdlException { public void testCheckPropertiesWithInvalidBalanceType() { // 测试无效的balance_type Map properties = Maps.newHashMap(); - properties.put(ComputeGroup.BALANCE_TYPE, "invalid_type"); + properties.put(CloudComputeGroupMeta.BALANCE_TYPE, "invalid_type"); Assertions.assertThrows(DdlException.class, () -> { computeGroup.checkProperties(properties); @@ -72,11 +73,11 @@ public void testCheckPropertiesWithInvalidBalanceType() { public void testCheckPropertiesWithValidTimeout() throws DdlException { // 测试有效的timeout Map properties = Maps.newHashMap(); - properties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); - properties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "300"); + properties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + properties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "300"); computeGroup.checkProperties(properties); - properties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "1"); + properties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "1"); computeGroup.checkProperties(properties); } @@ -84,13 +85,13 @@ public void testCheckPropertiesWithValidTimeout() throws DdlException { public void testCheckPropertiesWithInvalidTimeout() { // 测试无效的timeout Map properties = Maps.newHashMap(); - properties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "-1"); + properties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "-1"); Assertions.assertThrows(DdlException.class, () -> { computeGroup.checkProperties(properties); }); - properties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "invalid"); + properties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "invalid"); Assertions.assertThrows(DdlException.class, () -> { computeGroup.checkProperties(properties); }); @@ -111,62 +112,62 @@ public void testCheckPropertiesWithUnsupportedProperty() { public void testModifyPropertiesWithDirectSwitch() throws DdlException { // 测试without_warmup类型，应该删除timeout Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, - String.valueOf(ComputeGroup.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT)); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, + String.valueOf(CloudComputeGroupMeta.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT)); // 先设置timeout到properties中 - computeGroup.getProperties().put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, - String.valueOf(ComputeGroup.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT)); - Assertions.assertTrue(computeGroup.getProperties().containsKey(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT)); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, + String.valueOf(CloudComputeGroupMeta.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT)); + Assertions.assertTrue(computeGroup.getProperties().containsKey(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT)); computeGroup.modifyProperties(inputProperties); // 验证timeout被删除 - Assertions.assertFalse(computeGroup.getProperties().containsKey(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT)); + Assertions.assertFalse(computeGroup.getProperties().containsKey(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT)); } @Test public void testModifyPropertiesWithSyncCache() throws DdlException { // 测试sync_cache类型，应该删除timeout Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, - String.valueOf(ComputeGroup.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT)); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, + String.valueOf(CloudComputeGroupMeta.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT)); // 先设置timeout到properties中 - computeGroup.getProperties().put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, - String.valueOf(ComputeGroup.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT)); - Assertions.assertTrue(computeGroup.getProperties().containsKey(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT)); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, + String.valueOf(CloudComputeGroupMeta.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT)); + Assertions.assertTrue(computeGroup.getProperties().containsKey(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT)); computeGroup.modifyProperties(inputProperties); // 验证timeout被删除 - Assertions.assertFalse(computeGroup.getProperties().containsKey(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT)); + Assertions.assertFalse(computeGroup.getProperties().containsKey(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT)); } @Test public void testCheckPropertiesWithBalanceTypeTransition() throws DdlException { - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); computeGroup.checkProperties(inputProperties); } @Test public void testCheckPropertiesWithWarmupCacheToWarmupCache() throws DdlException { - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); computeGroup.checkProperties(inputProperties); } @Test public void testCheckPropertiesWithDirectSwitchToDirectSwitch() throws DdlException { // 测试从direct_switch转换到direct_switch，不需要设置timeout - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); computeGroup.checkProperties(inputProperties); } @@ -174,34 +175,34 @@ public void testCheckPropertiesWithDirectSwitchToDirectSwitch() throws DdlExcept public void testModifyPropertiesWithWarmupCacheAndExistingTimeout() throws DdlException { // 测试async_warmup类型，已存在timeout，不应该添加默认值 Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "600"); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "600"); // 先设置timeout到properties中 - computeGroup.getProperties().put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); - String originalTimeout = computeGroup.getProperties().get(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); + String originalTimeout = computeGroup.getProperties().get(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT); computeGroup.modifyProperties(inputProperties); // 验证timeout没有被修改, 这里的意思是用户已经设置过timeout了，就不应该被覆盖 - Assertions.assertEquals(originalTimeout, computeGroup.getProperties().get(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT)); + Assertions.assertEquals(originalTimeout, computeGroup.getProperties().get(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT)); } @Test public void testModifyPropertiesWithWarmupCacheAndNoTimeout() throws DdlException { // 测试async_warmup类型，不存在timeout，应该添加默认值 Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); // 确保properties中没有timeout - computeGroup.getProperties().remove(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT); - Assertions.assertFalse(computeGroup.getProperties().containsKey(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT)); + computeGroup.getProperties().remove(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT); + Assertions.assertFalse(computeGroup.getProperties().containsKey(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT)); computeGroup.modifyProperties(inputProperties); // 验证默认值被添加 - Assertions.assertTrue(computeGroup.getProperties().containsKey(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT)); - Assertions.assertEquals(String.valueOf(ComputeGroup.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT), - computeGroup.getProperties().get(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT)); + Assertions.assertTrue(computeGroup.getProperties().containsKey(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT)); + Assertions.assertEquals(String.valueOf(CloudComputeGroupMeta.DEFAULT_BALANCE_WARM_UP_TASK_TIMEOUT), + computeGroup.getProperties().get(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT)); } @Test @@ -233,56 +234,56 @@ public void testModifyPropertiesWithEmptyInput() throws DdlException { @Test public void testCheckPropertiesWithSyncCacheToWarmupCache() throws DdlException { - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); computeGroup.checkProperties(inputProperties); } @Test public void testValidateTimeoutRestrictionWithNoCurrentBalanceType() throws DdlException { // 测试当前没有设置balance_type的情况 - computeGroup.getProperties().remove(ComputeGroup.BALANCE_TYPE); + computeGroup.getProperties().remove(CloudComputeGroupMeta.BALANCE_TYPE); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); computeGroup.checkProperties(inputProperties); } @Test public void testValidateTimeoutRestrictionWithCurrentWarmupCache() throws DdlException { // 测试当前balance_type是warmup_cache的情况 - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); computeGroup.checkProperties(inputProperties); } @Test public void testValidateTimeoutRestrictionWithDirectSwitchToWarmupCache() throws DdlException { // 测试从direct_switch转换到warmup_cache并设置timeout - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); computeGroup.checkProperties(inputProperties); } @Test public void testValidateTimeoutRestrictionWithSyncCacheToWarmupCacheWithTimeout() throws DdlException { // 测试从sync_cache转换到warmup_cache并设置timeout - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.ASYNC_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); computeGroup.checkProperties(inputProperties); } @Test public void testValidateTimeoutRestrictionWithDirectSwitchAndOnlyTimeout() { // 测试当前是direct_switch，仅设置timeout应该失败 - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); Assertions.assertThrows(DdlException.class, () -> { computeGroup.checkProperties(inputProperties); @@ -292,9 +293,9 @@ public void testValidateTimeoutRestrictionWithDirectSwitchAndOnlyTimeout() { @Test public void testValidateTimeoutRestrictionWithSyncCacheAndOnlyTimeout() { // 测试当前是sync_cache，仅设置timeout应该失败 - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); Assertions.assertThrows(DdlException.class, () -> { computeGroup.checkProperties(inputProperties); @@ -304,10 +305,10 @@ public void testValidateTimeoutRestrictionWithSyncCacheAndOnlyTimeout() { @Test public void testValidateTimeoutRestrictionWithDirectSwitchToSyncCacheAndTimeout() { // 测试从direct_switch转换到sync_cache并设置timeout应该失败 - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.SYNC_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); Assertions.assertThrows(DdlException.class, () -> { computeGroup.checkProperties(inputProperties); @@ -317,10 +318,10 @@ public void testValidateTimeoutRestrictionWithDirectSwitchToSyncCacheAndTimeout( @Test public void testValidateTimeoutRestrictionWithDirectSwitchToSameAndTimeout() { // 测试从direct_switch转换到direct_switch并设置timeout应该失败 - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); - inputProperties.put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); - inputProperties.put(ComputeGroup.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); + inputProperties.put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + inputProperties.put(CloudComputeGroupMeta.BALANCE_WARM_UP_TASK_TIMEOUT, "500"); Assertions.assertThrows(DdlException.class, () -> { computeGroup.checkProperties(inputProperties); @@ -330,7 +331,7 @@ public void testValidateTimeoutRestrictionWithDirectSwitchToSameAndTimeout() { @Test public void testValidateTimeoutRestrictionWithNoInputTimeout() throws DdlException { // 测试输入中没有timeout的情况 - computeGroup.getProperties().put(ComputeGroup.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); + computeGroup.getProperties().put(CloudComputeGroupMeta.BALANCE_TYPE, BalanceTypeEnum.WITHOUT_WARMUP.getValue()); Map inputProperties = Maps.newHashMap(); inputProperties.put("other_property", "value"); Assertions.assertThrows(DdlException.class, () -> { diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudInstanceStatusCheckerTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudInstanceStatusCheckerTest.java index 95eb739de64867..fe4060d2ab44e0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudInstanceStatusCheckerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudInstanceStatusCheckerTest.java @@ -102,7 +102,7 @@ public void testSyncInstanceCreatesVirtualComputeGroup() { new CloudInstanceStatusChecker(cloudSystemInfoService).runAfterCatalogReady(); - ComputeGroup virtualComputeGroup = cloudSystemInfoService.getComputeGroupById("vcg_id"); + CloudComputeGroupMeta virtualComputeGroup = cloudSystemInfoService.getComputeGroupById("vcg_id"); Assertions.assertNotNull(virtualComputeGroup); Assertions.assertTrue(virtualComputeGroup.isVirtual()); Assertions.assertEquals("vcg", virtualComputeGroup.getName()); @@ -129,17 +129,17 @@ public void testSyncInstanceCreatesVirtualComputeGroupAndCancelsTableLevelLoadEv try (MockedStatic mockedCloudSystemInfoService = Mockito.mockStatic(CloudSystemInfoService.class, Mockito.CALLS_REAL_METHODS)) { mockedCloudSystemInfoService.when(() -> CloudSystemInfoService.updateFileCacheJobIds( - Mockito.any(ComputeGroup.class), Mockito.anyList())).thenAnswer(invocation -> null); + Mockito.any(CloudComputeGroupMeta.class), Mockito.anyList())).thenAnswer(invocation -> null); new CloudInstanceStatusChecker(cloudSystemInfoService).runAfterCatalogReady(); mockedCloudSystemInfoService.verify(() -> CloudSystemInfoService.updateFileCacheJobIds( - Mockito.any(ComputeGroup.class), Mockito.anyList())); + Mockito.any(CloudComputeGroupMeta.class), Mockito.anyList())); } finally { logger.removeAppender(appender); appender.stop(); } - ComputeGroup virtualComputeGroup = cloudSystemInfoService.getComputeGroupById("vcg_id"); + CloudComputeGroupMeta virtualComputeGroup = cloudSystemInfoService.getComputeGroupById("vcg_id"); Assertions.assertNotNull(virtualComputeGroup); Assertions.assertTrue(virtualComputeGroup.isVirtual()); Assertions.assertFalse(virtualComputeGroup.isNeedRebuildFileCache()); @@ -174,7 +174,7 @@ public void testSyncInstanceCreatesVirtualComputeGroupAndCancelsTableLevelLoadEv private void addComputeGroup(String computeGroupId, String computeGroupName) { cloudSystemInfoService.addComputeGroup(computeGroupId, - new ComputeGroup(computeGroupId, computeGroupName, ComputeGroup.ComputeTypeEnum.COMPUTE)); + new CloudComputeGroupMeta(computeGroupId, computeGroupName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE)); } private Cloud.GetInstanceResponse instanceResponseWithVirtualComputeGroup() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java index c37bd8a76cdf0f..7638bfa774d581 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceProxyTest.java @@ -34,19 +34,37 @@ import java.util.concurrent.atomic.AtomicInteger; public class MetaServiceProxyTest { + private static final int CPU_CORES = Runtime.getRuntime().availableProcessors(); + private String originEndpoint; private long originReconnectIntervalMs; private long originRetryCnt; + private boolean originRateLimitEnabled; + private int originRateLimitDefaultQpsPerCore; + private String originRateLimitQpsPerCoreConfig; + private int originRateLimitBurstSeconds; + private long originRateLimitWaitTimeoutMs; @Before public void setUp() { originEndpoint = Config.meta_service_endpoint; originReconnectIntervalMs = Config.meta_service_rpc_reconnect_interval_ms; originRetryCnt = Config.meta_service_rpc_retry_cnt; + originRateLimitEnabled = Config.meta_service_rpc_rate_limit_enabled; + originRateLimitDefaultQpsPerCore = Config.meta_service_rpc_rate_limit_default_qps_per_core; + originRateLimitQpsPerCoreConfig = Config.meta_service_rpc_rate_limit_qps_per_core_config; + originRateLimitBurstSeconds = Config.meta_service_rpc_rate_limit_burst_seconds; + originRateLimitWaitTimeoutMs = Config.meta_service_rpc_rate_limit_wait_timeout_ms; Config.meta_service_endpoint = "127.0.0.1:12345"; Config.meta_service_rpc_reconnect_interval_ms = 0; Config.meta_service_rpc_retry_cnt = 1; + Config.meta_service_rpc_rate_limit_enabled = false; + Config.meta_service_rpc_rate_limit_default_qps_per_core = 10; + Config.meta_service_rpc_rate_limit_qps_per_core_config = ""; + Config.meta_service_rpc_rate_limit_burst_seconds = 1; + Config.meta_service_rpc_rate_limit_wait_timeout_ms = 0; + MetaServiceProxy.resetMetaServiceRpcRateLimitForTest(); } @After @@ -54,6 +72,12 @@ public void tearDown() { Config.meta_service_endpoint = originEndpoint; Config.meta_service_rpc_reconnect_interval_ms = originReconnectIntervalMs; Config.meta_service_rpc_retry_cnt = originRetryCnt; + Config.meta_service_rpc_rate_limit_enabled = originRateLimitEnabled; + Config.meta_service_rpc_rate_limit_default_qps_per_core = originRateLimitDefaultQpsPerCore; + Config.meta_service_rpc_rate_limit_qps_per_core_config = originRateLimitQpsPerCoreConfig; + Config.meta_service_rpc_rate_limit_burst_seconds = originRateLimitBurstSeconds; + Config.meta_service_rpc_rate_limit_wait_timeout_ms = originRateLimitWaitTimeoutMs; + MetaServiceProxy.resetMetaServiceRpcRateLimitForTest(); } @Test @@ -216,6 +240,172 @@ public void testExecuteRequestFailureAfterTooBusyRetries() throws RpcException { Mockito.verify(client, Mockito.never()).shutdown(Mockito.anyBoolean()); } + @Test + public void testExecuteRequestRateLimitedWithoutRetryOrShutdown() throws RpcException { + enableRateLimit(1, "", 1, 0); + MetaServiceProxy proxy = new MetaServiceProxy(); + MetaServiceClient client = mockNormalClient(); + putClient(proxy, client); + + MetaServiceProxy.MetaServiceClientWrapper wrapper = Deencapsulation.getField(proxy, "w"); + AtomicInteger callCount = new AtomicInteger(); + Cloud.GetVersionResponse okResponse = okGetVersionResponse(); + for (int i = 0; i < CPU_CORES; i++) { + wrapper.executeRequest("limited", (ignored) -> { + callCount.incrementAndGet(); + return okResponse; + }, Cloud.GetVersionResponse::getStatus); + } + + try { + wrapper.executeRequest("limited", (ignored) -> { + callCount.incrementAndGet(); + return okResponse; + }, Cloud.GetVersionResponse::getStatus); + Assert.fail("should throw RpcException"); + } catch (RpcException e) { + Assert.assertTrue(e.getMessage().contains("meta service rpc rate limited")); + } + + Assert.assertEquals(CPU_CORES, callCount.get()); + Mockito.verify(client, Mockito.never()).shutdown(Mockito.anyBoolean()); + } + + @Test + public void testRateLimitSharedBetweenProxies() throws RpcException { + enableRateLimit(1, "", 1, 0); + MetaServiceProxy firstProxy = new MetaServiceProxy(); + MetaServiceProxy secondProxy = new MetaServiceProxy(); + putClient(firstProxy, mockNormalClient()); + putClient(secondProxy, mockNormalClient()); + + MetaServiceProxy.MetaServiceClientWrapper firstWrapper = Deencapsulation.getField(firstProxy, "w"); + MetaServiceProxy.MetaServiceClientWrapper secondWrapper = Deencapsulation.getField(secondProxy, "w"); + Cloud.GetVersionResponse okResponse = okGetVersionResponse(); + for (int i = 0; i < CPU_CORES; i++) { + firstWrapper.executeRequest("shared", (ignored) -> okResponse, Cloud.GetVersionResponse::getStatus); + } + + try { + secondWrapper.executeRequest("shared", (ignored) -> okResponse, Cloud.GetVersionResponse::getStatus); + Assert.fail("should throw RpcException"); + } catch (RpcException e) { + Assert.assertTrue(e.getMessage().contains("meta service rpc rate limited")); + } + } + + @Test + public void testGetInstanceRateLimitedBeforeRpc() throws RpcException { + enableRateLimit(1, "", 1, 0); + MetaServiceProxy proxy = new MetaServiceProxy(); + MetaServiceClient client = mockNormalClient(); + putClient(proxy, client); + consumeRateLimitPermits(proxy, "getInstance"); + + try { + proxy.getInstance(Cloud.GetInstanceRequest.newBuilder().build()); + Assert.fail("should throw RpcException"); + } catch (RpcException e) { + Assert.assertTrue(e.getMessage().contains("meta service rpc rate limited")); + } + + Mockito.verify(client, Mockito.never()).getInstance(Mockito.any()); + Mockito.verify(client, Mockito.never()).shutdown(Mockito.anyBoolean()); + } + + @Test + public void testGetVisibleVersionAsyncRateLimitedBeforeRpc() throws RpcException { + enableRateLimit(1, "", 1, 0); + MetaServiceProxy proxy = new MetaServiceProxy(); + MetaServiceClient client = mockNormalClient(); + putClient(proxy, client); + consumeRateLimitPermits(proxy, "getPartitionVersion"); + + try { + proxy.getVisibleVersionAsync(Cloud.GetVersionRequest.newBuilder().build()); + Assert.fail("should throw RpcException"); + } catch (RpcException e) { + Assert.assertTrue(e.getMessage().contains("meta service rpc rate limited")); + } + + Mockito.verify(client, Mockito.never()).getVisibleVersionAsync(Mockito.any()); + Mockito.verify(client, Mockito.never()).shutdown(Mockito.anyBoolean()); + } + + @Test + public void testBatchGetVisibleVersionAsyncConsumesMultipleRateLimitPermits() throws RpcException { + enableRateLimit(1, "", 1, 0); + MetaServiceProxy proxy = new MetaServiceProxy(); + MetaServiceClient client = mockNormalClient(); + putClient(proxy, client); + SettableFuture future = SettableFuture.create(); + Mockito.when(client.getVisibleVersionAsync(Mockito.any())).thenReturn(future); + + proxy.getVisibleVersionAsync(buildBatchTableVersionRequest(CPU_CORES)); + + try { + proxy.getVisibleVersionAsync(Cloud.GetVersionRequest.newBuilder() + .setIsTableVersion(true) + .build()); + Assert.fail("should throw RpcException"); + } catch (RpcException e) { + Assert.assertTrue(e.getMessage().contains("meta service rpc rate limited")); + } + Mockito.verify(client, Mockito.times(1)).getVisibleVersionAsync(Mockito.any()); + } + + private void enableRateLimit(int defaultQpsPerCore, String qpsPerCoreConfig, int burstSeconds, + long waitTimeoutMs) { + Config.meta_service_rpc_rate_limit_enabled = true; + Config.meta_service_rpc_rate_limit_default_qps_per_core = defaultQpsPerCore; + Config.meta_service_rpc_rate_limit_qps_per_core_config = qpsPerCoreConfig; + Config.meta_service_rpc_rate_limit_burst_seconds = burstSeconds; + Config.meta_service_rpc_rate_limit_wait_timeout_ms = waitTimeoutMs; + MetaServiceProxy.resetMetaServiceRpcRateLimitForTest(); + } + + private void putClient(MetaServiceProxy proxy, MetaServiceClient client) { + Map serviceMap = Deencapsulation.getField(proxy, "serviceMap"); + serviceMap.put(Config.meta_service_endpoint, client); + } + + private void consumeRateLimitPermits(MetaServiceProxy proxy, String methodName) throws RpcException { + MetaServiceProxy.MetaServiceClientWrapper wrapper = Deencapsulation.getField(proxy, "w"); + Cloud.GetVersionResponse okResponse = okGetVersionResponse(); + for (int i = 0; i < CPU_CORES; i++) { + wrapper.executeRequest(methodName, (ignored) -> okResponse, Cloud.GetVersionResponse::getStatus); + } + } + + private Cloud.GetVersionRequest buildBatchPartitionVersionRequest(int partitionNum) { + Cloud.GetVersionRequest.Builder builder = Cloud.GetVersionRequest.newBuilder() + .setBatchMode(true); + for (int i = 0; i < partitionNum; i++) { + builder.addDbIds(1L); + builder.addTableIds(1L); + builder.addPartitionIds(i); + } + return builder.build(); + } + + private Cloud.GetVersionRequest buildBatchTableVersionRequest(int tableNum) { + Cloud.GetVersionRequest.Builder builder = Cloud.GetVersionRequest.newBuilder() + .setBatchMode(true) + .setIsTableVersion(true); + for (int i = 0; i < tableNum; i++) { + builder.addDbIds(1L); + builder.addTableIds(i); + } + return builder.build(); + } + + private Cloud.GetVersionResponse okGetVersionResponse() { + return Cloud.GetVersionResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK)) + .build(); + } + private MetaServiceClient mockNormalClient() { MetaServiceClient client = Mockito.mock(MetaServiceClient.class); Mockito.when(client.isNormalState()).thenReturn(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceRpcRateLimiterTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceRpcRateLimiterTest.java new file mode 100644 index 00000000000000..e54db40f595ec6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/rpc/MetaServiceRpcRateLimiterTest.java @@ -0,0 +1,275 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.rpc; + +import org.apache.doris.common.Config; +import org.apache.doris.common.ConfigBase; +import org.apache.doris.common.MetaServiceRpcRateLimitConfigValidator; +import org.apache.doris.rpc.RpcException; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.concurrent.TimeUnit; + +public class MetaServiceRpcRateLimiterTest { + private static final int CPU_CORES = Runtime.getRuntime().availableProcessors(); + + private boolean originRateLimitEnabled; + private int originRateLimitDefaultQpsPerCore; + private String originRateLimitQpsPerCoreConfig; + private int originRateLimitBurstSeconds; + private long originRateLimitWaitTimeoutMs; + + private MetaServiceRpcRateLimiter rateLimiter; + + @Before + public void setUp() { + originRateLimitEnabled = Config.meta_service_rpc_rate_limit_enabled; + originRateLimitDefaultQpsPerCore = Config.meta_service_rpc_rate_limit_default_qps_per_core; + originRateLimitQpsPerCoreConfig = Config.meta_service_rpc_rate_limit_qps_per_core_config; + originRateLimitBurstSeconds = Config.meta_service_rpc_rate_limit_burst_seconds; + originRateLimitWaitTimeoutMs = Config.meta_service_rpc_rate_limit_wait_timeout_ms; + + rateLimiter = new MetaServiceRpcRateLimiter(); + enableRateLimit(1, "", 1, 0); + } + + @After + public void tearDown() { + Config.meta_service_rpc_rate_limit_enabled = originRateLimitEnabled; + Config.meta_service_rpc_rate_limit_default_qps_per_core = originRateLimitDefaultQpsPerCore; + Config.meta_service_rpc_rate_limit_qps_per_core_config = originRateLimitQpsPerCoreConfig; + Config.meta_service_rpc_rate_limit_burst_seconds = originRateLimitBurstSeconds; + Config.meta_service_rpc_rate_limit_wait_timeout_ms = originRateLimitWaitTimeoutMs; + rateLimiter.reset(); + } + + @Test + public void testMethodRateLimitCanBeDisabledByConfig() throws RpcException { + enableRateLimit(1, "unlimited:0", 1, 0); + + for (int i = 0; i <= CPU_CORES; i++) { + rateLimiter.acquire("unlimited"); + } + } + + @Test + public void testRateLimitDisabledBySwitch() throws RpcException { + consumePermits("disabledSwitch", CPU_CORES); + assertRateLimited("disabledSwitch"); + + Config.meta_service_rpc_rate_limit_enabled = false; + rateLimiter.acquire("disabledSwitch"); + } + + @Test + public void testMethodOverrideQpsTakesEffect() throws RpcException { + enableRateLimit(1, "fast:2", 1, 0); + + consumePermits("normal", CPU_CORES); + assertRateLimited("normal"); + + consumePermits("fast", CPU_CORES * 2); + assertRateLimited("fast"); + } + + @Test + public void testRateLimitIsIsolatedBetweenMethods() throws RpcException { + consumePermits("firstMethod", CPU_CORES); + assertRateLimited("firstMethod"); + + rateLimiter.acquire("secondMethod"); + } + + @Test + public void testWeightedAcquireConsumesMultiplePermits() throws RpcException { + rateLimiter.acquire("weighted", CPU_CORES); + + assertRateLimited("weighted"); + } + + @Test + public void testWeightedAcquireLargerThanTimeoutCapacityIsCappedAndSucceeds() throws RpcException { + rateLimiter.acquire("largeWeighted", CPU_CORES + 1); + + assertRateLimited("largeWeighted"); + } + + @Test + public void testWeightedAcquireUsesTimeoutCapacity() throws RpcException { + enableRateLimit(1, "", 1, 3000); + + long waitNs = rateLimiter.acquire("timeoutCapacity", CPU_CORES * 2); + + Assert.assertTrue(waitNs > 0); + } + + @Test + public void testDynamicConfigUpdate() throws RpcException { + consumePermits("dynamic", CPU_CORES); + assertRateLimited("dynamic"); + + Config.meta_service_rpc_rate_limit_default_qps_per_core = 0; + rateLimiter.acquire("dynamic"); + } + + @Test + public void testBurstWindow() throws RpcException { + enableRateLimit(1, "burst:1", 2, 0); + + consumePermits("burst", CPU_CORES * 2); + assertRateLimited("burst"); + } + + @Test + public void testWaitTimeoutRejectsWithoutWaitingForNextRefreshPeriod() throws RpcException { + enableRateLimit(1, "", 3, 1); + consumePermits("waitTimeout", CPU_CORES * 3); + + long startTimeMs = System.currentTimeMillis(); + assertRateLimited("waitTimeout"); + Assert.assertTrue(System.currentTimeMillis() - startTimeMs < TimeUnit.SECONDS.toMillis(1)); + } + + @Test + public void testAcquireReturnsActualWaitTime() throws RpcException { + enableRateLimit(1, "", 1, 2000); + consumePermits("wait", CPU_CORES); + + long waitNs = rateLimiter.acquire("wait"); + + Assert.assertTrue(waitNs > 0); + } + + @Test + public void testInvalidQpsConfig() throws Exception { + Field field = Config.class.getField("meta_service_rpc_rate_limit_qps_per_core_config"); + MetaServiceRpcRateLimitConfigValidator.QpsConfigHandler handler = + new MetaServiceRpcRateLimitConfigValidator.QpsConfigHandler(); + + assertConfigHandlerRejects(handler, field, "invalid", "Invalid"); + assertConfigHandlerRejects(handler, field, "method:", "Invalid"); + assertConfigHandlerRejects(handler, field, ":1", "Invalid"); + assertConfigHandlerRejects(handler, field, "method:abc", "Invalid"); + assertConfigHandlerRejects(handler, field, "method:1;method:2", "Duplicate"); + + Config.meta_service_rpc_rate_limit_qps_per_core_config = "invalid"; + assertRpcException("invalid meta_service_rpc_rate_limit_qps_per_core_config", + () -> rateLimiter.acquire("invalid")); + + Config.meta_service_rpc_rate_limit_qps_per_core_config = "method:1;method:2"; + assertRpcException("invalid meta_service_rpc_rate_limit_qps_per_core_config", + () -> rateLimiter.acquire("method")); + } + + @Test + public void testInvalidBurstSecondsConfigHandler() throws Exception { + Field field = Config.class.getField("meta_service_rpc_rate_limit_burst_seconds"); + MetaServiceRpcRateLimitConfigValidator.PositiveIntConfigHandler handler = + new MetaServiceRpcRateLimitConfigValidator.PositiveIntConfigHandler(); + + assertConfigHandlerRejects(handler, field, "0", "must be positive"); + assertConfigHandlerRejects(handler, field, "-1", "must be positive"); + + handler.handle(field, " 2 "); + Assert.assertEquals(2, Config.meta_service_rpc_rate_limit_burst_seconds); + } + + @Test + public void testInvalidBurstSeconds() throws RpcException { + enableRateLimit(1, "", 0, 0); + + assertRpcException("meta_service_rpc_rate_limit_burst_seconds must be positive", + () -> rateLimiter.acquire("invalidBurst")); + } + + @Test + public void testInvalidWaitTimeoutMsConfigHandler() throws Exception { + Field field = Config.class.getField("meta_service_rpc_rate_limit_wait_timeout_ms"); + MetaServiceRpcRateLimitConfigValidator.NonNegativeLongConfigHandler handler = + new MetaServiceRpcRateLimitConfigValidator.NonNegativeLongConfigHandler(); + + assertConfigHandlerRejects(handler, field, "-1", "must be non-negative"); + + handler.handle(field, " 0 "); + Assert.assertEquals(0, Config.meta_service_rpc_rate_limit_wait_timeout_ms); + } + + @Test + public void testInvalidWaitTimeoutMs() throws RpcException { + enableRateLimit(1, "", 1, -1); + + assertRpcException("meta_service_rpc_rate_limit_wait_timeout_ms must be non-negative", + () -> rateLimiter.acquire("invalidWaitTimeout")); + } + + @Test + public void testLimitForPeriodOverflow() throws RpcException { + enableRateLimit(Integer.MAX_VALUE, "", 2, 0); + + assertRpcException("meta service rpc rate limit is too large", + () -> rateLimiter.acquire("overflow")); + } + + private void enableRateLimit(int defaultQpsPerCore, String qpsPerCoreConfig, int burstSeconds, + long waitTimeoutMs) { + Config.meta_service_rpc_rate_limit_enabled = true; + Config.meta_service_rpc_rate_limit_default_qps_per_core = defaultQpsPerCore; + Config.meta_service_rpc_rate_limit_qps_per_core_config = qpsPerCoreConfig; + Config.meta_service_rpc_rate_limit_burst_seconds = burstSeconds; + Config.meta_service_rpc_rate_limit_wait_timeout_ms = waitTimeoutMs; + rateLimiter.reset(); + } + + private void consumePermits(String methodName, int permitNum) throws RpcException { + for (int i = 0; i < permitNum; i++) { + rateLimiter.acquire(methodName); + } + } + + private void assertConfigHandlerRejects(ConfigBase.ConfHandler handler, Field field, String config, + String expectedMessage) throws Exception { + try { + handler.handle(field, config); + Assert.fail("should throw exception"); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains(expectedMessage)); + } + } + + private void assertRateLimited(String methodName) throws RpcException { + assertRpcException("meta service rpc rate limited", () -> rateLimiter.acquire(methodName)); + } + + private void assertRpcException(String expectedMessage, RpcCall rpcCall) throws RpcException { + try { + rpcCall.run(); + Assert.fail("should throw RpcException"); + } catch (RpcException e) { + Assert.assertTrue(e.getMessage().contains(expectedMessage)); + } + } + + private interface RpcCall { + void run() throws RpcException; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java index 83f62b351eec8d..28a4decd6be456 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java @@ -19,8 +19,8 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.catalog.CloudComputeGroupMeta; import org.apache.doris.cloud.catalog.CloudEnv; -import org.apache.doris.cloud.catalog.ComputeGroup; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.rpc.MetaServiceProxy; import org.apache.doris.common.Config; @@ -71,7 +71,7 @@ public void testGetPhysicalClusterPhysicalCluster() { //public void testGetPhysicalClusterEmptyVirtualCluster() { // infoService = new CloudSystemInfoService(); // String vcgName = "v_cluster_1"; - // ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + // CloudComputeGroupMeta vcg = new CloudComputeGroupMeta("id1", vcgName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); // infoService.addComputeGroup(vcgName, vcg); // String res = infoService.getPhysicalCluster(vcgName); @@ -86,14 +86,14 @@ public void testGetPhysicalClusterEmptyCluster() { String pcgName1 = "p_cluster_1"; String pcgName2 = "p_cluster_2"; - ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta vcg = new CloudComputeGroupMeta("id1", vcgName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(pcgName1); policy.setStandbyComputeGroup(pcgName2); vcg.setPolicy(policy); - ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg1 = new CloudComputeGroupMeta("id2", pcgName1, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg2 = new CloudComputeGroupMeta("id3", pcgName2, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(vcgName, vcg); infoService.addComputeGroup(pcgName1, pcg1); infoService.addComputeGroup(pcgName2, pcg2); @@ -111,14 +111,14 @@ public void testGetPhysicalClusterStandbyAvailable() { String pcgName1 = "p_cluster_1"; String pcgName2 = "p_cluster_2"; - ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta vcg = new CloudComputeGroupMeta("id1", vcgName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(pcgName1); policy.setStandbyComputeGroup(pcgName2); vcg.setPolicy(policy); - ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg1 = new CloudComputeGroupMeta("id2", pcgName1, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg2 = new CloudComputeGroupMeta("id3", pcgName2, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(vcgName, vcg); infoService.addComputeGroup(pcgName1, pcg1); infoService.addComputeGroup(pcgName2, pcg2); @@ -148,14 +148,14 @@ public void testGetPhysicalClusterActiveAvailable() { String pcgName1 = "p_cluster_1"; String pcgName2 = "p_cluster_2"; - ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta vcg = new CloudComputeGroupMeta("id1", vcgName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(pcgName1); policy.setStandbyComputeGroup(pcgName2); vcg.setPolicy(policy); - ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg1 = new CloudComputeGroupMeta("id2", pcgName1, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg2 = new CloudComputeGroupMeta("id3", pcgName2, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(vcgName, vcg); infoService.addComputeGroup(pcgName1, pcg1); infoService.addComputeGroup(pcgName2, pcg2); @@ -185,14 +185,14 @@ public void testGetPhysicalClusterActive3AliveBe() { String pcgName1 = "p_cluster_1"; String pcgName2 = "p_cluster_2"; - ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta vcg = new CloudComputeGroupMeta("id1", vcgName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(pcgName1); policy.setStandbyComputeGroup(pcgName2); vcg.setPolicy(policy); - ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg1 = new CloudComputeGroupMeta("id2", pcgName1, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg2 = new CloudComputeGroupMeta("id3", pcgName2, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(vcgName, vcg); infoService.addComputeGroup(pcgName1, pcg1); infoService.addComputeGroup(pcgName2, pcg2); @@ -234,14 +234,14 @@ public void testGetPhysicalClusterStandby3AliveBe() { String pcgName1 = "p_cluster_1"; String pcgName2 = "p_cluster_2"; - ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta vcg = new CloudComputeGroupMeta("id1", vcgName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(pcgName1); policy.setStandbyComputeGroup(pcgName2); vcg.setPolicy(policy); - ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg1 = new CloudComputeGroupMeta("id2", pcgName1, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg2 = new CloudComputeGroupMeta("id3", pcgName2, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(vcgName, vcg); infoService.addComputeGroup(pcgName1, pcg1); infoService.addComputeGroup(pcgName2, pcg2); @@ -283,14 +283,14 @@ public void testGetPhysicalClusterSwitchActiveStandbyMetric() throws Exception { String pcgName1 = "p_cluster_1"; String pcgName2 = "p_cluster_2"; - ComputeGroup vcg = new ComputeGroup(vcgId, vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta vcg = new CloudComputeGroupMeta(vcgId, vcgName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(pcgName1); policy.setStandbyComputeGroup(pcgName2); policy.setUnhealthyNodeThresholdPercent(100); vcg.setPolicy(policy); - ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg2 = new CloudComputeGroupMeta("id3", pcgName2, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(vcgId, vcg); infoService.clusterNameToId.put(pcgName1, "id2"); infoService.addComputeGroup("id3", pcg2); @@ -342,14 +342,14 @@ public void testGetPhysicalClusterActive1AliveBe2DeadBe() { String pcgName1 = "p_cluster_1"; String pcgName2 = "p_cluster_2"; - ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta vcg = new CloudComputeGroupMeta("id1", vcgName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(pcgName1); policy.setStandbyComputeGroup(pcgName2); vcg.setPolicy(policy); - ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg1 = new CloudComputeGroupMeta("id2", pcgName1, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg2 = new CloudComputeGroupMeta("id3", pcgName2, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(vcgName, vcg); infoService.addComputeGroup(pcgName1, pcg1); infoService.addComputeGroup(pcgName2, pcg2); @@ -395,15 +395,15 @@ public void testIsStandByComputeGroup() { String pcgName2 = "p_cluster_2"; String pcgName3 = "p_cluster_3"; - ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta vcg = new CloudComputeGroupMeta("id1", vcgName, CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(pcgName1); policy.setStandbyComputeGroup(pcgName2); vcg.setPolicy(policy); - ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); - ComputeGroup pcg3 = new ComputeGroup("id4", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg1 = new CloudComputeGroupMeta("id2", pcgName1, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg2 = new CloudComputeGroupMeta("id3", pcgName2, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta pcg3 = new CloudComputeGroupMeta("id4", pcgName2, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(vcgName, vcg); infoService.addComputeGroup(pcgName1, pcg1); infoService.addComputeGroup(pcgName2, pcg2); @@ -427,7 +427,7 @@ public void testGetMinPipelineExecutorSizeWithEmptyCluster() { String clusterId = "test_cluster_id"; // Mock an empty cluster (no backends) - ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta(clusterId, clusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(clusterId, cg); // Set ConnectContext to select the cluster @@ -449,7 +449,7 @@ public void testGetMinPipelineExecutorSizeWithSingleBackend() { String clusterId = "test_cluster_id"; // Setup cluster - ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta(clusterId, clusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(clusterId, cg); // Add a backend with pipeline executor size = 8 @@ -483,7 +483,7 @@ public void testGetMinPipelineExecutorSizeWithMultipleBackends() { String clusterId = "test_cluster_id"; // Setup cluster - ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta(clusterId, clusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(clusterId, cg); // Add multiple backends with different pipeline executor sizes @@ -534,7 +534,7 @@ public void testGetMinPipelineExecutorSizeWithZeroSizeBackends() { String clusterId = "test_cluster_id"; // Setup cluster - ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta(clusterId, clusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(clusterId, cg); // Add backends with zero and positive pipeline executor sizes @@ -585,7 +585,7 @@ public void testGetMinPipelineExecutorSizeWithAllZeroSizeBackends() { String clusterId = "test_cluster_id"; // Setup cluster - ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta(clusterId, clusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(clusterId, cg); // Add backends with only zero or negative pipeline executor sizes @@ -645,7 +645,7 @@ public void testGetMinPipelineExecutorSizeWithMixedValidInvalidBackends() { String clusterId = "mixed_cluster_id"; // Setup cluster - ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta(clusterId, clusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(clusterId, cg); // Add backends with mixed valid and invalid pipeline executor sizes @@ -708,7 +708,7 @@ public void testGetMinPipelineExecutorSizeWithLargeValues() { String clusterId = "large_cluster_id"; // Setup cluster - ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta(clusterId, clusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(clusterId, cg); // Add backends with large pipeline executor sizes @@ -759,7 +759,7 @@ public void testGetMinPipelineExecutorSizeConsistency() { String clusterId = "consistency_cluster_id"; // Setup cluster - ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta(clusterId, clusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(clusterId, cg); // Add backends with same pipeline executor sizes @@ -800,11 +800,11 @@ public void testGetMinPipelineExecutorSizeWithMultipleComputeGroups() { String cluster2Id = "cluster2_id"; // Setup cluster1 - ComputeGroup cg1 = new ComputeGroup(cluster1Id, cluster1Name, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg1 = new CloudComputeGroupMeta(cluster1Id, cluster1Name, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(cluster1Id, cg1); // Setup cluster2 - ComputeGroup cg2 = new ComputeGroup(cluster2Id, cluster2Name, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg2 = new CloudComputeGroupMeta(cluster2Id, cluster2Name, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(cluster2Id, cg2); // Add backends to cluster1 with smaller pipeline executor sizes @@ -872,20 +872,20 @@ public void testGetMinPipelineExecutorSizeWithVirtualComputeGroup() { String otherClusterId = "other_cluster_id"; // Setup virtual cluster - ComputeGroup virtualCg = new ComputeGroup(virtualClusterId, virtualClusterName, - ComputeGroup.ComputeTypeEnum.VIRTUAL); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta virtualCg = new CloudComputeGroupMeta(virtualClusterId, virtualClusterName, + CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup(physicalClusterName); virtualCg.setPolicy(policy); infoService.addComputeGroup(virtualClusterId, virtualCg); // Setup physical cluster - ComputeGroup physicalCg = new ComputeGroup(physicalClusterId, physicalClusterName, - ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta physicalCg = new CloudComputeGroupMeta(physicalClusterId, physicalClusterName, + CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(physicalClusterId, physicalCg); // Setup other cluster - ComputeGroup otherCg = new ComputeGroup(otherClusterId, otherClusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta otherCg = new CloudComputeGroupMeta(otherClusterId, otherClusterName, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(otherClusterId, otherCg); // Add backends to physical cluster @@ -972,11 +972,11 @@ public void testGetMinPipelineExecutorSizeWithConnectContext() { String cluster2Id = "ctx_cluster2_id"; // Setup cluster1 - ComputeGroup cg1 = new ComputeGroup(cluster1Id, cluster1Name, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg1 = new CloudComputeGroupMeta(cluster1Id, cluster1Name, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(cluster1Id, cg1); // Setup cluster2 - ComputeGroup cg2 = new ComputeGroup(cluster2Id, cluster2Name, ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg2 = new CloudComputeGroupMeta(cluster2Id, cluster2Name, CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); infoService.addComputeGroup(cluster2Id, cg2); // Add backends to cluster1 with smaller pipeline executor sizes diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java index bcc5b41c3e8465..a6db5409e897f3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgrTest.java @@ -36,8 +36,10 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.UserException; +import org.apache.doris.load.routineload.RLTaskTxnCommitAttachment; import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.apache.doris.transaction.TransactionState; +import org.apache.doris.transaction.TxnStateChangeCallback; import com.google.common.collect.Lists; import org.junit.After; @@ -45,6 +47,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.jupiter.api.Assertions; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -306,6 +309,53 @@ public void testAbortTransaction() throws Exception { } } + @Test + public void testAbortRoutineLoadTransactionWithAttachment() throws Exception { + long transactionId = 123534; + long jobId = 1001; + RLTaskTxnCommitAttachment attachment = new RLTaskTxnCommitAttachment( + Cloud.RLTaskTxnCommitAttachmentPB.newBuilder() + .setJobId(jobId) + .setTaskId(Cloud.UniqueIdPB.newBuilder().setHi(1).setLo(2)) + .setProgress(Cloud.RoutineLoadProgressPB.newBuilder()) + .setFirstErrorMsg("invalid source row") + .build()); + TxnStateChangeCallback callback = Mockito.mock(TxnStateChangeCallback.class); + Mockito.when(callback.getId()).thenReturn(jobId); + masterTransMgr.getCallbackFactory().addCallback(callback); + + MetaServiceProxy mockProxy = Mockito.mock(MetaServiceProxy.class); + try (MockedStatic mockedStatic = Mockito.mockStatic(MetaServiceProxy.class)) { + mockedStatic.when(MetaServiceProxy::getInstance).thenReturn(mockProxy); + Mockito.doAnswer(invocation -> { + Cloud.AbortTxnRequest request = invocation.getArgument(0); + Assert.assertTrue(request.hasCommitAttachment()); + Assert.assertEquals("invalid source row", request.getCommitAttachment() + .getRlTaskTxnCommitAttachment().getFirstErrorMsg()); + return AbortTxnResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setTxnInfo(buildTxnInfo(transactionId).toBuilder() + .setStatus(Cloud.TxnStatusPB.TXN_STATUS_ABORTED) + .setReason("data quality error") + .setCommitAttachment(request.getCommitAttachment())) + .build(); + }).when(mockProxy).abortTxn(Mockito.any()); + + masterTransMgr.abortTransaction(CatalogTestUtil.testDbId1, transactionId, + "data quality error", attachment, Lists.newArrayList()); + + ArgumentCaptor txnStateCaptor = ArgumentCaptor.forClass(TransactionState.class); + Mockito.verify(callback).afterAborted(txnStateCaptor.capture(), Mockito.eq(true), + Mockito.eq("data quality error")); + RLTaskTxnCommitAttachment callbackAttachment = + (RLTaskTxnCommitAttachment) txnStateCaptor.getValue().getTxnCommitAttachment(); + Assert.assertEquals("invalid source row", callbackAttachment.getFirstErrorMsg()); + } finally { + masterTransMgr.getCallbackFactory().removeCallback(jobId); + } + } + @Test public void testAbortTransactionByLabel() throws Exception { MetaServiceProxy mockProxy = Mockito.mock(MetaServiceProxy.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/ColocationGroupProcDirTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/ColocationGroupProcDirTest.java index bfb53825947cb8..3e08cc822379e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/ColocationGroupProcDirTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/ColocationGroupProcDirTest.java @@ -86,7 +86,6 @@ public void testLocalColocationGroupDetailKeepsTagColumns() throws Exception { @Test public void testCloudColocationGroupDetailWithoutTag() throws Exception { - String originDeployMode = Config.deploy_mode; createTable("CREATE TABLE colocate_t1 (k INT) DISTRIBUTED BY HASH(k) BUCKETS 2 " + "PROPERTIES ('replication_num' = '1', 'colocate_with' = 'g1')"); createTable("CREATE TABLE colocate_t2 (k INT) DISTRIBUTED BY HASH(k) BUCKETS 2 " @@ -99,22 +98,20 @@ public void testCloudColocationGroupDetailWithoutTag() throws Exception { ColocateTableIndex colocateTableIndex = Mockito.spy(Env.getCurrentColocateIndex()); Mockito.doReturn(Maps.>>newHashMap()).when(colocateTableIndex) .getBackendsPerBucketSeq(groupId); - Config.deploy_mode = "cloud"; - try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + try (MockedStatic mockedConfig = Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + mockedConfig.when(Config::isCloudMode).thenReturn(true); mockedEnv.when(Env::getCurrentColocateIndex).thenReturn(colocateTableIndex); ProcNodeInterface node = new ColocationGroupProcDir().lookup(groupId.toString()); ProcResult result = node.fetchResult(); Assertions.assertEquals(Lists.newArrayList("BucketIndex", "BackendIds"), result.getColumnNames()); Assertions.assertFalse(result.getRows().isEmpty()); Assertions.assertTrue(result.getRows().stream().anyMatch(row -> row.size() == 2 && !row.get(1).isEmpty())); - } finally { - Config.deploy_mode = originDeployMode; } } @Test public void testCloudGlobalColocationGroupDetailFallback() throws Exception { - String originDeployMode = Config.deploy_mode; createTable("CREATE TABLE global_colocate_t1 (k INT) DISTRIBUTED BY HASH(k) BUCKETS 2 " + "PROPERTIES ('replication_num' = '1', 'colocate_with' = '__global__g1')"); @@ -125,22 +122,20 @@ public void testCloudGlobalColocationGroupDetailFallback() throws Exception { ColocateTableIndex colocateTableIndex = Mockito.spy(Env.getCurrentColocateIndex()); Mockito.doReturn(Maps.>>newHashMap()).when(colocateTableIndex) .getBackendsPerBucketSeq(groupId); - Config.deploy_mode = "cloud"; - try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + try (MockedStatic mockedConfig = Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + mockedConfig.when(Config::isCloudMode).thenReturn(true); mockedEnv.when(Env::getCurrentColocateIndex).thenReturn(colocateTableIndex); ProcNodeInterface node = new ColocationGroupProcDir().lookup(groupId.toString()); ProcResult result = node.fetchResult(); Assertions.assertEquals(Lists.newArrayList("BucketIndex", "BackendIds"), result.getColumnNames()); Assertions.assertFalse(result.getRows().isEmpty()); Assertions.assertTrue(result.getRows().stream().anyMatch(row -> row.size() == 2 && !row.get(1).isEmpty())); - } finally { - Config.deploy_mode = originDeployMode; } } @Test public void testCloudColocationGroupDetailFallbackSkipsUnusableFirstTable() throws Exception { - String originDeployMode = Config.deploy_mode; createTable("CREATE TABLE colocate_t5 (k INT) DISTRIBUTED BY HASH(k) BUCKETS 2 " + "PROPERTIES ('replication_num' = '1', 'colocate_with' = 'g3')"); createTable("CREATE TABLE colocate_t6 (k INT) DISTRIBUTED BY HASH(k) BUCKETS 2 " @@ -153,8 +148,9 @@ public void testCloudColocationGroupDetailFallbackSkipsUnusableFirstTable() thro ColocateTableIndex colocateTableIndex = Mockito.spy(Env.getCurrentColocateIndex()); Mockito.doReturn(Maps.>>newHashMap()).when(colocateTableIndex) .getBackendsPerBucketSeq(groupId); - Config.deploy_mode = "cloud"; - try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + try (MockedStatic mockedConfig = Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + mockedConfig.when(Config::isCloudMode).thenReturn(true); mockedEnv.when(Env::getCurrentColocateIndex).thenReturn(colocateTableIndex); ProcNodeInterface node = new ColocationGroupProcDir().lookup(groupId.toString()); ProcResult result = node.fetchResult(); @@ -163,20 +159,18 @@ public void testCloudColocationGroupDetailFallbackSkipsUnusableFirstTable() thro Assertions.assertTrue(result.getRows().stream().anyMatch(row -> row.size() == 2 && !row.get(1).isEmpty())); } finally { db.registerTable(table1); - Config.deploy_mode = originDeployMode; } } @Test public void testCloudColocationGroupReplicaAllocationIsNull() throws Exception { - String originDeployMode = Config.deploy_mode; createTable("CREATE TABLE colocate_t3 (k INT) DISTRIBUTED BY HASH(k) BUCKETS 2 " + "PROPERTIES ('replication_num' = '1', 'colocate_with' = 'g2')"); createTable("CREATE TABLE colocate_t4 (k INT) DISTRIBUTED BY HASH(k) BUCKETS 2 " + "PROPERTIES ('replication_num' = '1', 'colocate_with' = 'g2')"); - Config.deploy_mode = "cloud"; - try { + try (MockedStatic mockedConfig = Mockito.mockStatic(Config.class, Mockito.CALLS_REAL_METHODS)) { + mockedConfig.when(Config::isCloudMode).thenReturn(true); ProcResult result = new ColocationGroupProcDir().fetchResult(); int groupNameIdx = ColocationGroupProcDir.TITLE_NAMES.indexOf("GroupName"); int replicaAllocIdx = ColocationGroupProcDir.TITLE_NAMES.indexOf("ReplicaAllocation"); @@ -185,8 +179,6 @@ public void testCloudColocationGroupReplicaAllocationIsNull() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("can not find colocate group test.g2")); Assertions.assertEquals("null", groupRow.get(replicaAllocIdx)); - } finally { - Config.deploy_mode = originDeployMode; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java index afe40a68dc8c16..f6e8d5a677d7f7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/profile/SummaryProfileTest.java @@ -17,6 +17,8 @@ package org.apache.doris.common.profile; +import org.apache.doris.common.Config; + import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -78,6 +80,34 @@ public void testPreloadExternalMetadataTimeCounter() { Assertions.assertEquals("20ms", profile.getPrettyNereidsPreloadExternalMetadataTime()); } + @Test + public void testMetaVersionRateLimitWaitTime() { + String originalCloudUniqueId = Config.cloud_unique_id; + Config.cloud_unique_id = "test_cloud"; + try { + SummaryProfile profile = new SummaryProfile(); + profile.addGetPartitionVersionTime(1_000_000); + profile.addGetTableVersionTime(2_000_000); + profile.addGetMetaVersionRateLimitWaitTime(1_000_000); + profile.addGetMetaVersionRateLimitWaitTime(2_000_000); + + profile.update(ImmutableMap.of()); + + RuntimeProfile executionSummary = profile.getExecutionSummary(); + Assertions.assertEquals(3_000_000, profile.getGetMetaVersionRateLimitWaitTime()); + Assertions.assertEquals("3.0ms", executionSummary.getInfoString( + SummaryProfile.GET_META_VERSION_RATE_LIMIT_WAIT_TIME)); + String metaTime = profile.getMetaTime(); + Assertions.assertTrue(metaTime.contains("\"get_partition_version_time_ms\":1")); + Assertions.assertTrue(metaTime.contains("\"get_table_version_time_ms\":2")); + Assertions.assertTrue(metaTime.contains("\"get_meta_version_rate_limit_wait_time_ms\":3")); + Assertions.assertFalse(new SummaryProfile().getMetaTime().contains( + "get_meta_version_rate_limit_wait_time_ms")); + } finally { + Config.cloud_unique_id = originalCloudUniqueId; + } + } + @Test public void testExternalTableMetaSummary() { SummaryProfile profile = new SummaryProfile(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java index 5bba968b49af41..5d2099c5c61d0e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java @@ -17,11 +17,19 @@ package org.apache.doris.common.util; +import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.httpv2.meta.MetaBaseAction; +import org.apache.doris.system.SystemInfoService.HostInfo; import org.junit.After; import org.junit.Assert; import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.net.HttpURLConnection; +import java.util.Map; public class HttpURLUtilTest { @@ -30,6 +38,57 @@ public void tearDown() { Config.enable_https = false; Config.http_port = 8030; Config.https_port = 8050; + Config.fe_meta_auth_token = ""; + } + + @Test + public void testNodeIdentHeadersIncludeClusterToken() throws Exception { + Config.fe_meta_auth_token = "cluster-token"; + Env env = Mockito.mock(Env.class); + Mockito.when(env.getSelfNode()).thenReturn(new HostInfo("127.0.0.1", 9010)); + + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getServingEnv).thenReturn(env); + + Map headers = HttpURLUtil.getNodeIdentHeaders(); + + Assert.assertEquals("127.0.0.1", headers.get(Env.CLIENT_NODE_HOST_KEY)); + Assert.assertEquals("9010", headers.get(Env.CLIENT_NODE_PORT_KEY)); + Assert.assertEquals("cluster-token", headers.get(MetaBaseAction.TOKEN)); + } + } + + @Test + public void testNodeIdentHeadersOmitTokenWhenNotConfigured() throws Exception { + Config.fe_meta_auth_token = ""; + Env env = Mockito.mock(Env.class); + Mockito.when(env.getSelfNode()).thenReturn(new HostInfo("127.0.0.1", 9010)); + + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getServingEnv).thenReturn(env); + + Map headers = HttpURLUtil.getNodeIdentHeaders(); + + Assert.assertEquals("127.0.0.1", headers.get(Env.CLIENT_NODE_HOST_KEY)); + Assert.assertFalse(headers.containsKey(MetaBaseAction.TOKEN)); + } + } + + @Test + public void testNodeIdentConnectionIncludesClusterToken() throws Exception { + Config.fe_meta_auth_token = "cluster-token"; + Env env = Mockito.mock(Env.class); + Mockito.when(env.getSelfNode()).thenReturn(new HostInfo("127.0.0.1", 9010)); + + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getServingEnv).thenReturn(env); + + HttpURLConnection connection = HttpURLUtil.getConnectionWithNodeIdent("http://127.0.0.1:8030/info"); + + Assert.assertEquals("127.0.0.1", connection.getRequestProperty(Env.CLIENT_NODE_HOST_KEY)); + Assert.assertEquals("9010", connection.getRequestProperty(Env.CLIENT_NODE_PORT_KEY)); + Assert.assertEquals("cluster-token", connection.getRequestProperty(MetaBaseAction.TOKEN)); + } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/InternalHttpsUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/InternalHttpsUtilsTest.java index 47f41c4858d555..d082420d8397e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/InternalHttpsUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/InternalHttpsUtilsTest.java @@ -28,18 +28,18 @@ public class InternalHttpsUtilsTest { - private String originalCertPath; + private String originalKeyStorePath; @Before public void setUp() throws Exception { - originalCertPath = Config.mysql_ssl_default_ca_certificate; + originalKeyStorePath = Config.key_store_path; // Reset the cached SSLContext before each test so tests are independent. resetCachedSslContext(); } @After public void tearDown() throws Exception { - Config.mysql_ssl_default_ca_certificate = originalCertPath; + Config.key_store_path = originalKeyStorePath; resetCachedSslContext(); } @@ -51,14 +51,15 @@ private void resetCachedSslContext() throws Exception { @Test public void testGetSslContextThrowsWhenCertMissing() { - Config.mysql_ssl_default_ca_certificate = "/non/existent/path/ca.p12"; + Config.key_store_path = "/non/existent/path/doris_ssl_certificate.keystore"; try { InternalHttpsUtils.getSslContext(); Assert.fail("Expected RuntimeException when cert file does not exist"); } catch (RuntimeException e) { // Error message must mention the cert path so operators know what to fix. Assert.assertTrue("Error message should contain cert path", - e.getMessage() != null && e.getMessage().contains("/non/existent/path/ca.p12")); + e.getMessage() != null + && e.getMessage().contains("/non/existent/path/doris_ssl_certificate.keystore")); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 3ecb3a72d6b023..9422f08812de2d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -28,6 +28,7 @@ import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.schema.external.TArrayField; import org.apache.doris.thrift.schema.external.TField; +import org.apache.doris.thrift.schema.external.TFieldPtr; import org.apache.doris.thrift.schema.external.TNestedField; import org.apache.doris.thrift.schema.external.TSchema; import org.apache.doris.thrift.schema.external.TStructField; @@ -254,6 +255,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col1.isAllowNull(), field1.isIsOptional()); Assert.assertEquals(col1.getType().toColumnTypeThrift(), field1.getType()); Assert.assertEquals(Arrays.asList("m_c1"), field1.getNameMapping()); + Assert.assertTrue(field1.isNameMappingIsAuthoritative()); Assert.assertEquals("7", field1.getInitialDefaultValue()); Assert.assertFalse(field1.isSetInitialDefaultValueIsBase64()); @@ -262,7 +264,72 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col2.isAllowNull(), field2.isIsOptional()); Assert.assertEquals(col2.getType().toColumnTypeThrift(), field2.getType()); Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); + Assert.assertTrue(field2.isNameMappingIsAuthoritative()); Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } + + @Test + public void testInitSchemaInfoForAllColumnSerializesNestedNonBinaryDefault() { + StructType structType = new StructType( + new StructField("added_int", Type.INT, "nested default", true)); + Column structColumn = new Column("s", structType, true); + structColumn.setUniqueId(10); + Column child = structColumn.getChildren().get(0); + child.setUniqueId(11); + child.setDefaultValueInfo(new Column("added_int", Type.INT, false, null, true, "7", "")); + TFileScanRangeParams params = new TFileScanRangeParams(); + + ExternalUtil.initSchemaInfoForAllColumn( + params, 1L, Collections.singletonList(structColumn), Collections.emptyMap()); + + TField childField = params.getHistorySchemaInfo().get(0).getRootField().getFields().get(0) + .getFieldPtr().getNestedField().getStructField().getFields().get(0).getFieldPtr(); + Assert.assertEquals("7", childField.getInitialDefaultValue()); + Assert.assertFalse(childField.isSetInitialDefaultValueIsBase64()); + } + + @Test + public void testInitSchemaInfoForAllColumnPreservesPartialNameMapping() { + TFileScanRangeParams params = new TFileScanRangeParams(); + Column mappedColumn = new Column("a", Type.INT, true); + mappedColumn.setUniqueId(1); + Column unmappedColumn = new Column("b", Type.INT, true); + unmappedColumn.setUniqueId(2); + + Map> nameMapping = new HashMap<>(); + nameMapping.put(mappedColumn.getUniqueId(), Collections.singletonList("a")); + ExternalUtil.initSchemaInfoForAllColumn( + params, 600L, Arrays.asList(mappedColumn, unmappedColumn), nameMapping); + + List fields = params.getHistorySchemaInfo().get(0).getRootField().getFields(); + Assert.assertEquals(Collections.singletonList("a"), fields.get(0).getFieldPtr().getNameMapping()); + Assert.assertTrue(fields.get(0).getFieldPtr().isNameMappingIsAuthoritative()); + Assert.assertTrue(fields.get(1).getFieldPtr().isSetNameMapping()); + Assert.assertTrue(fields.get(1).getFieldPtr().getNameMapping().isEmpty()); + Assert.assertTrue(fields.get(1).getFieldPtr().isNameMappingIsAuthoritative()); + } + + @Test + public void testInitSchemaInfoForAllColumnDistinguishesAbsentAndEmptyNameMapping() { + Column column = new Column("a", Type.INT, true); + column.setUniqueId(1); + + TFileScanRangeParams absentParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn( + absentParams, 700L, Collections.singletonList(column), Collections.emptyMap()); + TField absentField = absentParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertFalse(absentField.isSetNameMapping()); + Assert.assertFalse(absentField.isSetNameMappingIsAuthoritative()); + + TFileScanRangeParams emptyParams = new TFileScanRangeParams(); + ExternalUtil.initSchemaInfoForAllColumn(emptyParams, 701L, + Collections.singletonList(column), Collections.emptyMap(), true, Collections.emptyMap()); + TField emptyField = emptyParams.getHistorySchemaInfo().get(0) + .getRootField().getFields().get(0).getFieldPtr(); + Assert.assertTrue(emptyField.isSetNameMapping()); + Assert.assertTrue(emptyField.getNameMapping().isEmpty()); + Assert.assertTrue(emptyField.isNameMappingIsAuthoritative()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java index 752ee5195a4398..cf57b84f10be65 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java @@ -161,4 +161,5 @@ public void testUpdateRequiredSlotsPreservesInlineDefaultValueExpr() throws Exce Assert.assertTrue(updatedSlotInfo.isSetDefaultValueExpr()); Assert.assertSame(defaultExpr, updatedSlotInfo.getDefaultValueExpr()); } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index 0a798faff93fef..b4a1c6a5d1b235 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -188,6 +188,11 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedSpec).when(mockedIcebergTable).spec(); Mockito.doReturn(ImmutableMap.of()).when(mockedIcebergTable).specs(); Mockito.doReturn(icebergSchema).when(mockedIcebergTable).schema(); + // The scan now resolves initial defaults from the statement-pinned schema id, so the + // mocked table must expose the same historical-schema lookup as a real Iceberg table. + Mockito.doAnswer(invocation -> ImmutableMap.of( + mockedIcebergTable.schema().schemaId(), mockedIcebergTable.schema())) + .when(mockedIcebergTable).schemas(); // Mock newScan() chain used by IcebergScanNode.createTableScan() TableScan mockedTableScan = Mockito.mock(TableScan.class, Mockito.RETURNS_DEEP_STUBS); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index 937167c34c351d..f78c186d0d1b5c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.analysis.ColumnPath; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MapType; @@ -24,37 +25,61 @@ import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.Type; +import org.apache.doris.catalog.info.ColumnPosition; import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; public class IcebergMetadataOpsValidationTest { private IcebergMetadataOps ops; + private ExternalCatalog dorisCatalog; private Method validateForModifyColumnMethod; private Method validateForModifyComplexColumnMethod; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void setUp() throws Exception { - ExternalCatalog dorisCatalog = Mockito.mock(ExternalCatalog.class); + dorisCatalog = Mockito.mock(ExternalCatalog.class); Catalog icebergCatalog = Mockito.mock(Catalog.class, Mockito.withSettings().extraInterfaces(SupportsNamespaces.class)); Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); + Mockito.doReturn(Optional.empty()).when(dorisCatalog).getDbForReplay(Mockito.anyString()); ops = new IcebergMetadataOps(dorisCatalog, icebergCatalog); validateForModifyColumnMethod = IcebergMetadataOps.class.getDeclaredMethod( @@ -88,6 +113,15 @@ public void testValidateForModifyColumnSuccess() throws Throwable { invokeValidateForModifyColumn(column, currentCol); } + @Test + public void testValidateForModifyColumnRejectsComplexToPrimitive() { + Column column = new Column("struct_col", Type.INT, true); + NestedField currentCol = Types.NestedField.required(1, "struct_col", Types.StructType.of( + Types.NestedField.required(2, "value", Types.IntegerType.get()))); + assertUserException(() -> invokeValidateForModifyColumn(column, currentCol), + "Modify column type from complex to primitive is not supported: struct_col"); + } + @Test public void testValidateForModifyComplexColumnRejectsPrimitiveType() { Column column = new Column("arr_i", Type.INT, true); @@ -182,6 +216,1193 @@ public void testValidateForModifyComplexColumnSuccess() throws Throwable { invokeValidateForModifyComplexColumn(column, currentCol); } + @Test + public void testRejectUnsupportedIcebergTargetTypesBeforeUpdateSchema() { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + StructType unsupportedStruct = new StructType(new StructField("value", Type.LARGEINT)); + ArrayType unsupportedArray = ArrayType.create(Type.LARGEINT, true); + MapType unsupportedMap = new MapType(Type.STRING, Type.LARGEINT); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("info.new_field"), + new Column("new_field", Type.LARGEINT, true), null, 1L), + "is not supported for Iceberg column new_field"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), + new Column("metric", Type.LARGEINT, true), null, 1L), + "is not supported for Iceberg column info.metric"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), + new Column("child", unsupportedStruct, false), null, 1L), + "is not supported for Iceberg column info.child.value"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.events"), + new Column("events", unsupportedArray, false), null, 1L), + "is not supported for Iceberg column info.events.element"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.attrs"), + new Column("attrs", unsupportedMap, false), null, 1L), + "is not supported for Iceberg column info.attrs.value"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), + new Column("child", new StructType(new StructField("value", Type.BIGINT)), true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.events"), + new Column("events", ArrayType.create(Type.BIGINT, true), true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.attrs"), + new Column("attrs", new MapType(Type.STRING, Type.BIGINT), true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("info.child.value", Types.LongType.get(), null); + Mockito.verify(updateSchema).updateColumn("info.events.element", Types.LongType.get(), null); + Mockito.verify(updateSchema).updateColumn("info.attrs.value", Types.LongType.get(), null); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); + Mockito.verify(updateSchema, Mockito.times(3)).commit(); + } + + @Test + public void testComplexModifyPersistsDecodedStructMemberComment() throws Throwable { + Schema schema = new Schema(Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "payload", Types.StructType.of( + Types.NestedField.optional(3, "name", Types.StringType.get(), "old comment")))))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + String decodedComment = "owner's \"field\" C:\\tmp\\"; + Column column = new Column("payload", new StructType( + new StructField("name", Type.STRING, decodedComment, true)), true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), column, null, 1L); + } + + Mockito.verify(updateSchema).updateColumnDoc("info.payload.name", decodedComment); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testPrimitiveModifyPreservesOmittedCommentAndClearsExplicitEmptyComment() throws Throwable { + Schema schema = new Schema( + Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "metric", Types.IntegerType.get(), "metric doc"), + Types.NestedField.optional(3, "clear_me", Types.StringType.get(), "clear doc"))), + Types.NestedField.optional(4, "top_metric", Types.IntegerType.get(), "top metric doc")); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + Column clearComment = new Column("clear_me", Type.STRING, true, ""); + clearComment.setCommentSpecified(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), + new Column("metric", Type.BIGINT, true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.of("top_metric"), + new Column("top_metric", Type.BIGINT, true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.clear_me"), + clearComment, null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("info.metric", Types.LongType.get(), "metric doc"); + Mockito.verify(updateSchema).updateColumn("top_metric", Types.LongType.get(), "top metric doc"); + Mockito.verify(updateSchema).updateColumnDoc("info.clear_me", ""); + Mockito.verify(updateSchema, Mockito.times(3)).commit(); + } + + @Test + public void testFullStructModifyPreservesOmittedChildComments() throws Throwable { + Schema schema = new Schema(Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "payload", Types.StructType.of( + Types.NestedField.optional(3, "metric", Types.IntegerType.get(), "metric doc"), + Types.NestedField.optional(4, "clear_me", Types.StringType.get(), "clear doc"), + Types.NestedField.optional(5, "details", Types.StructType.of( + Types.NestedField.optional( + 6, "count", Types.IntegerType.get(), "count doc")), "details doc")), + "payload doc")))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + StructType detailsType = new StructType( + new StructField("count", Type.BIGINT, "", true, false)); + StructType payloadType = new StructType( + new StructField("metric", Type.BIGINT, "", true, false), + new StructField("clear_me", Type.STRING, "", true, true), + new StructField("details", detailsType, "", true, false)); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), + new Column("payload", payloadType, true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn( + "info.payload.metric", Types.LongType.get(), "metric doc"); + Mockito.verify(updateSchema).updateColumnDoc("info.payload.clear_me", ""); + Mockito.verify(updateSchema).updateColumn( + "info.payload.details.count", Types.LongType.get(), "count doc"); + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.eq("info.payload"), Mockito.nullable(String.class)); + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.eq("info.payload.details"), Mockito.nullable(String.class)); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), + new Column("metric", Type.BIGINT, true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("info.metric", Types.LongType.get(), null); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwable { + Schema schema = new Schema( + Types.NestedField.required(1, "Id", Types.IntegerType.get()), + Types.NestedField.required(2, "Payload", Types.StructType.of( + Types.NestedField.required(3, "Value", Types.IntegerType.get())))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("id"), + new Column("id", Type.BIGINT, true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.of("payload"), new Column("payload", + new StructType(new StructField("Value", Type.BIGINT)), true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("Id", Types.LongType.get(), null); + Mockito.verify(updateSchema).updateColumn("Payload.Value", Types.LongType.get(), null); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); + Mockito.verify(updateSchema, Mockito.times(2)).commit(); + } + + @Test + public void testTopLevelModifyDoesNotResolveQuotedComponentAsNestedPath() { + Schema schema = new Schema( + Types.NestedField.optional(1, "a", Types.StructType.of( + Types.NestedField.optional(2, "b", Types.IntegerType.get()))), + Types.NestedField.optional(3, "b", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), + new Column("a.b", Type.BIGINT, true), null, 1L), + "Column a.b does not exist"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testTopLevelModifyPreservesDottedTopLevelName() throws Throwable { + Schema schema = new Schema(Types.NestedField.optional(1, "a.b", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), + new Column("a.b", Type.BIGINT, true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("a.b", Types.LongType.get(), null); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws Throwable { + Schema schema = mappedPrimitiveSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(false); + Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(false); + + Column topUuid = new Column("top_uuid", Type.STRING, true); + topUuid.setNullableSpecified(true); + Column nestedUuid = new Column("uuid_value", Type.STRING, true); + nestedUuid.setNullableSpecified(true); + Column nestedTimestamp = new Column( + "tz_value", ScalarType.createDatetimeV2Type(6), true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), topUuid, ColumnPosition.FIRST, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.uuid_value"), nestedUuid, + new ColumnPosition("other"), 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.tz_value"), + nestedTimestamp, null, 1L); + } + + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.anyString(), Mockito.nullable(String.class)); + Mockito.verify(updateSchema).makeColumnOptional("top_uuid"); + Mockito.verify(updateSchema).makeColumnOptional("info.uuid_value"); + Mockito.verify(updateSchema).moveFirst("top_uuid"); + Mockito.verify(updateSchema).moveAfter("info.uuid_value", "info.other"); + Mockito.verify(updateSchema, Mockito.never()).updateColumn( + Mockito.anyString(), Mockito.any(org.apache.iceberg.types.Type.PrimitiveType.class), + Mockito.nullable(String.class)); + Mockito.verify(updateSchema, Mockito.times(3)).commit(); + } + + @Test + public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Throwable { + Schema schema = mappedPrimitiveSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(true); + Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), + new Column("top_uuid", ScalarType.createVarbinaryType(16), true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.tz_value"), + new Column("tz_value", ScalarType.createTimeStampTzType(6), true), null, 1L); + } + + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.anyString(), Mockito.nullable(String.class)); + Mockito.verify(updateSchema, Mockito.never()).updateColumn( + Mockito.anyString(), Mockito.any(org.apache.iceberg.types.Type.PrimitiveType.class), + Mockito.nullable(String.class)); + Mockito.verify(updateSchema, Mockito.times(2)).commit(); + } + + @Test + public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { + Schema schema = mappedComplexSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(true); + Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), + new Column("payload", mappedPayloadDorisType(Type.BIGINT, 8, + ScalarType.createVarbinaryType(4)), true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn( + "outer.payload.metric", Types.LongType.get(), null); + Mockito.verify(updateSchema, Mockito.times(1)).updateColumn( + Mockito.anyString(), Mockito.any(org.apache.iceberg.types.Type.PrimitiveType.class), + Mockito.nullable(String.class)); + Mockito.verify(updateSchema, Mockito.never()).updateColumnDoc( + Mockito.anyString(), Mockito.nullable(String.class)); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testComplexModifyRejectsChangedUnsupportedMappedChildrenBeforeUpdateSchema() { + Schema schema = mappedComplexSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(dorisCatalog.getEnableMappingVarbinary()).thenReturn(true); + Mockito.when(dorisCatalog.getEnableMappingTimestampTz()).thenReturn(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), + new Column("payload", mappedPayloadDorisType(Type.LARGEINT, 8, + ScalarType.createVarbinaryType(4)), true), null, 1L), + "Type largeint is not supported for Iceberg column outer.payload.metric"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), + new Column("payload", mappedPayloadDorisType(Type.INT, 16, + ScalarType.createVarbinaryType(4)), true), null, 1L), + "Type varbinary(16) is not supported for Iceberg column outer.payload.fixed_value"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), + new Column("payload", mappedPayloadDorisType(Type.INT, 8, + ScalarType.createVarbinaryType(8)), true), null, 1L), + "Cannot change MAP key type from varbinary(4) to varbinary(8)"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + // Iceberg schema columns are represented as keys in Doris, so the legacy API must not + // interpret isKey as an explicit KEY clause. + ops.modifyColumn(dorisTable, + new Column("id", Type.BIGINT, true, null, true, null, ""), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("id", Types.LongType.get(), ""); + Mockito.verify(updateSchema).makeColumnOptional("id"); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testLegacyComplexModifyDoesNotInferRecursiveNullableChanges() throws Throwable { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + Column column = new Column("info", new StructType( + new StructField("metric", Type.INT), + new StructField("child", new StructType(new StructField("value", Type.INT))), + new StructField("events", ArrayType.create(Type.INT, true)), + new StructField("attrs", new MapType(Type.STRING, Type.INT))), true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, column, null, 1L); + } + + Mockito.verify(updateSchema).makeColumnOptional("info"); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional("info.metric"); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional("info.child.value"); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional("info.events.element"); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional("info.attrs.value"); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throwable { + Schema schema = requiredNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + Column topLevelColumn = new Column("info", new StructType( + new StructField("metric", Type.BIGINT), + new StructField("child", new StructType(new StructField("value", Type.INT))), + new StructField("events", ArrayType.create(Type.INT, true)), + new StructField("attrs", new MapType(Type.STRING, Type.INT))), true); + topLevelColumn.setNullableSpecified(true); + Column nestedColumn = new Column("metric", Type.BIGINT, true); + nestedColumn.setNullableSpecified(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.of("info"), topLevelColumn, null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), nestedColumn, null, 1L); + } + + Mockito.verify(updateSchema).makeColumnOptional("info"); + Mockito.verify(updateSchema).makeColumnOptional("info.metric"); + Mockito.verify(updateSchema, Mockito.times(2)).commit(); + } + + @Test + public void testSchemaCommitConflictDoesNotPartiallyApplyComplexModify() throws Exception { + String dbName = "db"; + String tableName = "conflict_table"; + TableIdentifier tableIdentifier = TableIdentifier.of(dbName, tableName); + Map properties = new HashMap<>(); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, + temporaryFolder.newFolder("iceberg_warehouse").toURI().toString()); + + HadoopCatalog icebergCatalog = new HadoopCatalog(); + icebergCatalog.setConf(new Configuration()); + icebergCatalog.initialize("conflict_catalog", properties); + icebergCatalog.createNamespace(Namespace.of(dbName)); + Table staleTable = icebergCatalog.createTable(tableIdentifier, new Schema( + Types.NestedField.optional(1, "info", Types.StructType.of( + Types.NestedField.optional(2, "a", Types.IntegerType.get()), + Types.NestedField.optional(3, "b", Types.IntegerType.get()))))); + Table concurrentTable = icebergCatalog.loadTable(tableIdentifier); + + ExternalCatalog conflictDorisCatalog = Mockito.mock(ExternalCatalog.class); + AtomicBoolean conflictInjected = new AtomicBoolean(false); + Mockito.when(conflictDorisCatalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public void execute(Runnable task) { + Assert.assertTrue("schema commit should execute only once", conflictInjected.compareAndSet(false, true)); + concurrentTable.updateSchema().renameColumn("info.a", "concurrent_a").commit(); + task.run(); + } + }); + Mockito.when(conflictDorisCatalog.getProperties()).thenReturn(Collections.emptyMap()); + Mockito.doReturn(Optional.empty()).when(conflictDorisCatalog).getDbForReplay(Mockito.anyString()); + IcebergMetadataOps conflictOps = new IcebergMetadataOps(conflictDorisCatalog, icebergCatalog); + + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn(dbName); + Mockito.when(dorisTable.getRemoteName()).thenReturn(tableName); + StructType promotedInfo = new StructType( + new StructField("a", Type.BIGINT), + new StructField("b", Type.BIGINT)); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(staleTable); + + try { + conflictOps.modifyColumn(dorisTable, ColumnPath.of("info"), + new Column("info", promotedInfo, true), null, 1L); + Assert.fail("expected schema commit conflict"); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("Failed to modify column: info")); + Assert.assertTrue("expected Iceberg commit conflict but got " + e.getCause(), + e.getCause() instanceof CommitFailedException); + } + } + + Schema committedSchema = icebergCatalog.loadTable(tableIdentifier).schema(); + Assert.assertNull(committedSchema.findField("info.a")); + Assert.assertEquals(Types.IntegerType.get(), committedSchema.findType("info.concurrent_a")); + Assert.assertEquals(Types.IntegerType.get(), committedSchema.findType("info.b")); + Mockito.verify(conflictDorisCatalog, Mockito.never()).getDbForReplay(Mockito.anyString()); + + icebergCatalog.dropTable(tableIdentifier); + icebergCatalog.dropNamespace(Namespace.of(dbName)); + icebergCatalog.close(); + } + + @Test + public void testRenamePreservesNestedIdentifierFieldPaths() throws Exception { + String dbName = "db"; + String tableName = "identifier_table"; + TableIdentifier tableIdentifier = TableIdentifier.of(dbName, tableName); + Map properties = new HashMap<>(); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, + temporaryFolder.newFolder("identifier_warehouse").toURI().toString()); + + HadoopCatalog icebergCatalog = new HadoopCatalog(); + icebergCatalog.setConf(new Configuration()); + icebergCatalog.initialize("identifier_catalog", properties); + icebergCatalog.createNamespace(Namespace.of(dbName)); + Schema schema = new Schema(Arrays.asList( + Types.NestedField.required(1, "root", Types.StructType.of( + Types.NestedField.required(2, "child", Types.StructType.of( + Types.NestedField.required(3, "id", Types.IntegerType.get()), + Types.NestedField.optional(4, "value", Types.StringType.get())))))), + Collections.singleton(3)); + Table icebergTable = icebergCatalog.createTable(tableIdentifier, schema); + + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn(dbName); + Mockito.when(dorisTable.getRemoteName()).thenReturn(tableName); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child.id"), "renamed_id", 1L); + icebergTable.refresh(); + Assert.assertEquals(Collections.singleton("root.child.renamed_id"), + icebergTable.schema().identifierFieldNames()); + + ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child"), "renamed_child", 1L); + icebergTable.refresh(); + Assert.assertEquals(Collections.singleton("root.renamed_child.renamed_id"), + icebergTable.schema().identifierFieldNames()); + + ops.renameColumn(dorisTable, "root", "renamed_root", 1L); + icebergTable.refresh(); + Assert.assertEquals(Collections.singleton("renamed_root.renamed_child.renamed_id"), + icebergTable.schema().identifierFieldNames()); + Assert.assertEquals(3, icebergTable.schema().findField( + "renamed_root.renamed_child.renamed_id").fieldId()); + } + + icebergCatalog.dropTable(tableIdentifier); + icebergCatalog.dropNamespace(Namespace.of(dbName)); + icebergCatalog.close(); + } + + @Test + public void testRenameDoesNotRewriteDottedIdentifierSibling() throws Exception { + String dbName = "db"; + String tableName = "dotted_identifier_table"; + TableIdentifier tableIdentifier = TableIdentifier.of(dbName, tableName); + Map properties = new HashMap<>(); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, + temporaryFolder.newFolder("dotted_identifier_warehouse").toURI().toString()); + + HadoopCatalog icebergCatalog = new HadoopCatalog(); + icebergCatalog.setConf(new Configuration()); + icebergCatalog.initialize("dotted_identifier_catalog", properties); + icebergCatalog.createNamespace(Namespace.of(dbName)); + Schema schema = new Schema(Arrays.asList( + Types.NestedField.required(1, "a", Types.IntegerType.get()), + Types.NestedField.required(2, "a.b", Types.IntegerType.get())), + Collections.singleton(2)); + Table icebergTable = icebergCatalog.createTable(tableIdentifier, schema); + + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn(dbName); + Mockito.when(dorisTable.getRemoteName()).thenReturn(tableName); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.renameColumn(dorisTable, "a", "renamed", 1L); + icebergTable.refresh(); + Assert.assertNotNull(icebergTable.schema().findField("renamed")); + Assert.assertEquals(Collections.singleton("a.b"), icebergTable.schema().identifierFieldNames()); + Assert.assertEquals(2, icebergTable.schema().findField("a.b").fieldId()); + } + + icebergCatalog.dropTable(tableIdentifier); + icebergCatalog.dropNamespace(Namespace.of(dbName)); + icebergCatalog.close(); + } + + @Test + public void testNestedColumnOperationsRejectDefaultMetadata() { + Schema schema = new Schema(Types.NestedField.optional(1, "s", Types.StructType.of( + Types.NestedField.optional(2, "existing", Types.IntegerType.get())))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + Column nestedAddDefaultColumn = new Column("new_col", Type.BIGINT, false, null, true, "7", ""); + Column nestedDefaultColumn = new Column("existing", Type.BIGINT, false, null, true, "7", ""); + Column nestedOnUpdateColumn = Mockito.spy(new Column("existing", Type.BIGINT, true)); + Mockito.doReturn(true).when(nestedOnUpdateColumn).hasOnUpdateDefaultValue(); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), + nestedAddDefaultColumn, null, 1L), + "DEFAULT and ON UPDATE are not supported for Iceberg nested ADD COLUMN: s.new_col"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.existing"), + nestedDefaultColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: s.existing"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("s.existing"), + nestedOnUpdateColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: s.existing"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testTopLevelColumnOperationsRejectUnsupportedDefaultMetadata() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + Column defaultColumn = new Column("id", Type.BIGINT, false, null, true, "7", ""); + Column onUpdateColumn = Mockito.spy(new Column("id", Type.BIGINT, true)); + Column onUpdateAddColumn = Mockito.spy(new Column("new_col", Type.BIGINT, true)); + Mockito.doReturn(true).when(onUpdateColumn).hasOnUpdateDefaultValue(); + Mockito.doReturn(true).when(onUpdateAddColumn).hasOnUpdateDefaultValue(); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn(dorisTable, defaultColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: id"); + assertUserException(() -> ops.modifyColumn( + dorisTable, ColumnPath.of("id"), onUpdateColumn, null, 1L), + "Modifying default values is not supported for Iceberg columns: id"); + assertUserException(() -> ops.addColumn(dorisTable, onUpdateAddColumn, null, 1L), + "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); + assertUserException(() -> ops.addColumns( + dorisTable, Collections.singletonList(onUpdateAddColumn), 1L), + "ON UPDATE is not supported for Iceberg ADD COLUMN: new_col"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testUnsupportedPrimitiveModifyFailsBeforeUpdateSchema() { + Schema schema = new Schema( + Types.NestedField.required(1, "top_long", Types.LongType.get()), + Types.NestedField.required(2, "info", Types.StructType.of( + Types.NestedField.required(3, "metric", Types.LongType.get()), + Types.NestedField.required(4, "child", Types.StructType.of( + Types.NestedField.required(5, "value", Types.IntegerType.get())))))); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn( + dorisTable, ColumnPath.of("info"), new Column("info", Type.INT, true), null, 1L), + "Modify column type from complex to primitive is not supported: info"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), + new Column("child", Type.INT, true), null, 1L), + "Modify column type from complex to primitive is not supported: info.child"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("top_long"), + new Column("top_long", Type.INT, true), null, 1L), + "Cannot change column type: top_long: long -> int"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), + new Column("metric", Type.INT, true), null, 1L), + "Cannot change column type: info.metric: long -> int"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + Column keyColumn = new Column("id", Type.BIGINT, true, null, true, null, ""); + Column generatedColumn = Mockito.mock(Column.class); + Mockito.when(generatedColumn.getName()).thenReturn("id"); + Mockito.when(generatedColumn.isGeneratedColumn()).thenReturn(true); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn(dorisTable, keyColumn, null, 1L), + "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + assertUserException(() -> ops.addColumns(dorisTable, Collections.singletonList(keyColumn), 1L), + "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + assertUserException(() -> ops.modifyColumn( + dorisTable, ColumnPath.of("id"), keyColumn, null, 1L), + "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); + assertUserException(() -> ops.addColumns(dorisTable, + Collections.singletonList(generatedColumn), 1L), + "Generated columns are not supported for Iceberg ADD/MODIFY COLUMN"); + assertUserException(() -> ops.modifyColumn( + dorisTable, ColumnPath.of("id"), generatedColumn, null, 1L), + "Generated columns are not supported for Iceberg ADD/MODIFY COLUMN"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() { + Schema schema = mixedCaseNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + StructType infoType = new StructType( + new StructField("Metric", Type.INT), + new StructField("Label", Type.STRING), + new StructField("metric", Type.INT)); + ArrayType eventsType = ArrayType.create(new StructType( + new StructField("Score", Type.INT), + new StructField("score", Type.INT)), true); + MapType attrsType = new MapType(Type.STRING, new StructType( + new StructField("Code", Type.INT), + new StructField("code", Type.INT))); + StructType duplicateNewFieldsType = new StructType( + new StructField("Metric", Type.INT), + new StructField("Label", Type.STRING), + new StructField("Extra", Type.INT), + new StructField("EXTRA", Type.STRING)); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn( + dorisTable, new Column("info", infoType, true), null, 1L), + "Added struct field 'metric' conflicts with existing field"); + assertUserException(() -> ops.modifyColumn( + dorisTable, new Column("events", eventsType, true), null, 1L), + "Added struct field 'score' conflicts with existing field"); + assertUserException(() -> ops.modifyColumn( + dorisTable, new Column("attrs", attrsType, true), null, 1L), + "Added struct field 'code' conflicts with existing field"); + assertUserException(() -> ops.modifyColumn( + dorisTable, new Column("info", duplicateNewFieldsType, true), null, 1L), + "Added struct field 'extra' conflicts with existing field"); + } + + Mockito.verifyNoInteractions(updateSchema); + } + + @Test + public void testResolveNestedColumnPathSupportsStructArrayElementAndMapValue() throws Throwable { + Schema schema = nestedSchema(); + Assert.assertTrue(ops.resolveNestedColumnPath(schema, ColumnPath.fromDotName("s"), "add").isStructType()); + Assert.assertTrue(ops.resolveNestedColumnPath(schema, ColumnPath.fromDotName("arr.element"), "add") + .isStructType()); + Assert.assertTrue(ops.resolveNestedColumnPath(schema, ColumnPath.fromDotName("m.value"), "add") + .isStructType()); + } + + @Test + public void testResolveNestedColumnPathUsesCaseInsensitiveCanonicalIcebergPath() throws Throwable { + Schema schema = mixedCaseNestedSchema(); + Assert.assertEquals("Info.Metric", + ops.getCanonicalColumnPath(schema, ColumnPath.fromDotName("info.metric"), "modify")); + Assert.assertEquals("Events.element.Score", + ops.getCanonicalColumnPath(schema, ColumnPath.fromDotName("events.element.score"), "modify")); + Assert.assertEquals("Attrs.value.Code", + ops.getCanonicalColumnPath(schema, ColumnPath.fromDotName("attrs.value.code"), "modify")); + Assert.assertEquals("Info.Label", + ops.getPositionReferencePath(schema, ColumnPath.fromDotName("Info.NewField"), + new ColumnPosition("label"), "add")); + } + + @Test + public void testValidateNoCaseInsensitiveSiblingCollisionRejectsAddAndRenameTargets() { + Types.StructType parentType = mixedCaseNestedSchema().findField("Info").type().asStructType(); + ColumnPath parentPath = ColumnPath.fromDotName("Info"); + assertUserException(() -> ops.validateNoCaseInsensitiveSiblingCollision( + parentType, parentPath, "metric", null, "add"), + "Cannot add nested column 'Info.metric': conflicts with existing Iceberg field 'Info.Metric'"); + assertUserException(() -> ops.validateNoCaseInsensitiveSiblingCollision( + parentType, parentPath, "metric", parentType.field("Label"), "rename"), + "Cannot rename nested column 'Info.metric': conflicts with existing Iceberg field 'Info.Metric'"); + } + + @Test + public void testValidateNoCaseInsensitiveSiblingCollisionAllowsCaseOnlyRename() throws Throwable { + Types.StructType parentType = mixedCaseNestedSchema().findField("Info").type().asStructType(); + ops.validateNoCaseInsensitiveSiblingCollision(parentType, ColumnPath.fromDotName("Info"), + "metric", parentType.field("Metric"), "rename"); + } + + @Test + public void testTopLevelCaseInsensitiveCollisionsAndCaseOnlyRename() throws Throwable { + Schema schema = new Schema( + Types.NestedField.optional(1, "Id", Types.IntegerType.get()), + Types.NestedField.optional(2, "Label", Types.StringType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn( + dorisTable, new Column("id", Type.STRING, true), null, 1L), + "Cannot add column 'id': conflicts with existing Iceberg field 'Id'"); + assertUserException(() -> ops.addColumns(dorisTable, + Collections.singletonList(new Column("id", Type.STRING, true)), 1L), + "Cannot add column 'id': conflicts with existing Iceberg field 'Id'"); + assertUserException(() -> ops.addColumns(dorisTable, Arrays.asList( + new Column("new_field", Type.STRING, true), + new Column("NEW_FIELD", Type.STRING, true)), 1L), + "conflicts with another requested column (case-insensitive)"); + assertUserException(() -> ops.renameColumn(dorisTable, "label", "id", 1L), + "Cannot rename column 'id': conflicts with existing Iceberg field 'Id'"); + + ops.renameColumn(dorisTable, "id", "id", 1L); + } + + Mockito.verify(updateSchema).renameColumn("Id", "id"); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testReorderColumnsUsesCanonicalIcebergNames() throws Throwable { + Schema schema = new Schema( + Types.NestedField.optional(1, "Id", Types.IntegerType.get()), + Types.NestedField.optional(2, "Label", Types.StringType.get())); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.reorderColumns(dorisTable, Arrays.asList("label", "id"), 1L); + } + + Mockito.verify(updateSchema).moveFirst("Label"); + Mockito.verify(updateSchema).moveAfter("Id", "Label"); + Mockito.verify(updateSchema).commit(); + } + + @Test + public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throwable { + Schema schema = primitiveContainerSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + new Column("element", Type.BIGINT, true), null, 1L); + ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + new Column("value", Type.BIGINT, true), null, 1L); + } + + Mockito.verify(updateSchema).updateColumn("arr.element", Types.LongType.get(), null); + Mockito.verify(updateSchema).updateColumn("m.value", Types.LongType.get(), null); + Mockito.verify(updateSchema, Mockito.never()).makeColumnOptional(Mockito.anyString()); + Mockito.verify(updateSchema, Mockito.times(2)).commit(); + } + + @Test + public void testModifyColumnRejectsPositionForDirectArrayElementAndMapValue() { + Schema schema = primitiveContainerSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + new Column("element", Type.BIGINT, true), ColumnPosition.FIRST, 1L), + "Cannot apply column position to 'arr.element': parent column path 'arr' is not a struct"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + new Column("element", Type.BIGINT, true), new ColumnPosition("element"), 1L), + "Cannot apply column position to 'arr.element': parent column path 'arr' is not a struct"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + new Column("value", Type.BIGINT, true), ColumnPosition.FIRST, 1L), + "Cannot apply column position to 'm.value': parent column path 'm' is not a struct"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + new Column("value", Type.BIGINT, true), new ColumnPosition("value"), 1L), + "Cannot apply column position to 'm.value': parent column path 'm' is not a struct"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { + Schema schema = mixedCaseNestedSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + UpdateSchema updateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + Mockito.when(icebergTable.updateSchema()).thenReturn(updateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("info.metric"), + "struct comment", 1L); + ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("events.element.score"), + "array element comment", 1L); + ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("attrs.value.code"), + "map value comment", 1L); + } + + Mockito.verify(updateSchema).updateColumnDoc("Info.Metric", "struct comment"); + Mockito.verify(updateSchema).updateColumnDoc("Events.element.Score", "array element comment"); + Mockito.verify(updateSchema).updateColumnDoc("Attrs.value.Code", "map value comment"); + Mockito.verify(updateSchema, Mockito.times(3)).commit(); + } + + @Test + public void testRejectsCommentsOnDirectArrayElementAndMapValue() { + Schema schema = primitiveContainerSchema(); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(dorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(icebergTable.schema()).thenReturn(schema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.modifyColumnComment( + dorisTable, ColumnPath.fromDotName("arr.element"), "array element comment", 1L), + "Iceberg does not support comments on collection element or value fields: arr.element"); + assertUserException(() -> ops.modifyColumnComment( + dorisTable, ColumnPath.fromDotName("m.value"), "map value comment", 1L), + "Iceberg does not support comments on collection element or value fields: m.value"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + new Column("element", Type.BIGINT, true, "array element comment"), null, 1L), + "Iceberg does not support comments on collection element or value fields: arr.element"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + new Column("value", Type.BIGINT, true, "map value comment"), null, 1L), + "Iceberg does not support comments on collection element or value fields: m.value"); + Column arrayElementWithEmptyComment = new Column("element", Type.BIGINT, true, ""); + arrayElementWithEmptyComment.setCommentSpecified(true); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), + arrayElementWithEmptyComment, null, 1L), + "Iceberg does not support comments on collection element or value fields: arr.element"); + Column mapValueWithEmptyComment = new Column("value", Type.BIGINT, true, ""); + mapValueWithEmptyComment.setCommentSpecified(true); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("m.value"), + mapValueWithEmptyComment, null, 1L), + "Iceberg does not support comments on collection element or value fields: m.value"); + assertUserException(() -> ops.modifyColumnComment( + dorisTable, ColumnPath.fromDotName("m.key"), "map key comment", 1L), + "Cannot modify comment MAP key nested column"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testRejectsTopLevelRowLineageMutationsForV3Tables() { + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Table icebergTable = Mockito.mock(Table.class); + Mockito.when(icebergTable.properties()).thenReturn(Collections.singletonMap("format-version", "3")); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + + assertUserException(() -> ops.addColumn(dorisTable, + new Column("_row_id", Type.BIGINT, true), null, 1L), + "Cannot add Iceberg v3 reserved row lineage column: _row_id"); + assertUserException(() -> ops.addColumns(dorisTable, Collections.singletonList( + new Column("_last_updated_sequence_number", Type.BIGINT, true)), 1L), + "Cannot add Iceberg v3 reserved row lineage column: _last_updated_sequence_number"); + assertUserException(() -> ops.dropColumn(dorisTable, "_ROW_ID", 1L), + "Cannot drop Iceberg v3 reserved row lineage column: _ROW_ID"); + assertUserException(() -> ops.renameColumn(dorisTable, "_row_id", "renamed", 1L), + "Cannot rename Iceberg v3 reserved row lineage column: _row_id"); + assertUserException(() -> ops.renameColumn( + dorisTable, "id", "_last_updated_sequence_number", 1L), + "Cannot rename to Iceberg v3 reserved row lineage column: _last_updated_sequence_number"); + assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("_row_id"), + new Column("_row_id", Type.BIGINT, true), null, 1L), + "Cannot modify Iceberg v3 reserved row lineage column: _row_id"); + assertUserException(() -> ops.modifyColumnComment(dorisTable, + ColumnPath.of("_last_updated_sequence_number"), "comment", 1L), + "Cannot modify comment for Iceberg v3 reserved row lineage column: " + + "_last_updated_sequence_number"); + assertUserException(() -> ops.reorderColumns(dorisTable, + Arrays.asList("_row_id", "id"), 1L), + "Cannot reorder Iceberg v3 reserved row lineage column: _row_id"); + } + + Mockito.verify(icebergTable, Mockito.never()).updateSchema(); + } + + @Test + public void testAllowsV3NestedAndV2TopLevelRowLineageNames() throws Throwable { + Schema v3Schema = new Schema( + Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "s", Types.StructType.of( + Types.NestedField.optional(3, "source", Types.LongType.get()), + Types.NestedField.optional(4, "_row_id", Types.LongType.get())))); + ExternalTable v3DorisTable = Mockito.mock(ExternalTable.class); + Table v3IcebergTable = Mockito.mock(Table.class); + UpdateSchema v3UpdateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(v3DorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(v3IcebergTable.properties()).thenReturn(Collections.singletonMap("format-version", "3")); + Mockito.when(v3IcebergTable.schema()).thenReturn(v3Schema); + Mockito.when(v3IcebergTable.updateSchema()).thenReturn(v3UpdateSchema); + + Schema v2Schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + ExternalTable v2DorisTable = Mockito.mock(ExternalTable.class); + Table v2IcebergTable = Mockito.mock(Table.class); + UpdateSchema v2UpdateSchema = Mockito.mock(UpdateSchema.class); + Mockito.when(v2DorisTable.getRemoteDbName()).thenReturn("db"); + Mockito.when(v2IcebergTable.properties()).thenReturn(Collections.singletonMap("format-version", "2")); + Mockito.when(v2IcebergTable.schema()).thenReturn(v2Schema); + Mockito.when(v2IcebergTable.updateSchema()).thenReturn(v2UpdateSchema); + + try (MockedStatic mockedIcebergUtils = + Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v3DorisTable)).thenReturn(v3IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v2DorisTable)).thenReturn(v2IcebergTable); + + ops.addColumn(v3DorisTable, ColumnPath.fromDotName("s._last_updated_sequence_number"), + new Column("_last_updated_sequence_number", Type.BIGINT, true), null, 1L); + ops.renameColumn(v3DorisTable, ColumnPath.fromDotName("s.source"), "_last_updated_sequence_number", 1L); + ops.modifyColumn(v3DorisTable, ColumnPath.fromDotName("s._row_id"), + new Column("_row_id", Type.BIGINT, true), null, 1L); + ops.modifyColumnComment(v3DorisTable, ColumnPath.fromDotName("s._row_id"), "comment", 1L); + ops.dropColumn(v3DorisTable, ColumnPath.fromDotName("s._row_id"), 1L); + + ops.addColumn(v2DorisTable, new Column("_row_id", Type.BIGINT, true), null, 1L); + ops.renameColumn(v2DorisTable, "id", "_last_updated_sequence_number", 1L); + } + + Mockito.verify(v3UpdateSchema, Mockito.times(5)).commit(); + Mockito.verify(v2UpdateSchema, Mockito.times(2)).commit(); + } + + @Test + public void testResolveNestedColumnPathRejectsMapKey() { + assertUserException(() -> ops.resolveNestedColumnPath(nestedSchema(), ColumnPath.fromDotName("m.key.x"), + "modify"), + "Cannot modify MAP key nested column"); + } + + @Test + public void testResolveNestedColumnPathRejectsPrimitiveParent() { + assertUserException(() -> ops.resolveNestedColumnPath(nestedSchema(), ColumnPath.fromDotName("id.x"), + "modify"), + "Cannot resolve nested field under primitive column path"); + } + + @Test + public void testGetPositionReferencePathForNestedColumn() { + Assert.assertEquals("s.a", ops.getPositionReferencePath(ColumnPath.fromDotName("s.new_col"), + new ColumnPosition("a"))); + Assert.assertEquals("arr.element.x", ops.getPositionReferencePath( + ColumnPath.fromDotName("arr.element.new_col"), new ColumnPosition("x"))); + Assert.assertEquals("m.value.v", ops.getPositionReferencePath( + ColumnPath.fromDotName("m.value.new_col"), new ColumnPosition("v"))); + Assert.assertEquals("id", ops.getPositionReferencePath(ColumnPath.fromDotName("new_col"), + new ColumnPosition("id"))); + } + + @Test + public void testValidateNestedStructFieldSupportsStructArrayElementAndMapValueFields() throws Throwable { + Schema schema = nestedSchema(); + Assert.assertTrue(ops.validateNestedStructField(schema, ColumnPath.fromDotName("s.a"), "drop") + .isPrimitiveType()); + Assert.assertTrue(ops.validateNestedStructField(schema, ColumnPath.fromDotName("arr.element.x"), "drop") + .isPrimitiveType()); + Assert.assertTrue(ops.validateNestedStructField(schema, ColumnPath.fromDotName("m.value.v"), "rename") + .isPrimitiveType()); + } + + @Test + public void testValidateNestedStructFieldRejectsArrayElementAndMapValuePseudoFields() { + assertUserException(() -> ops.validateNestedStructField(nestedSchema(), ColumnPath.fromDotName("arr.element"), + "drop"), + "Parent column path 'arr' is not a struct"); + assertUserException(() -> ops.validateNestedStructField(nestedSchema(), ColumnPath.fromDotName("m.value"), + "rename"), + "Parent column path 'm' is not a struct"); + } + + @Test + public void testValidateNestedStructFieldRejectsMapKeyAndMissingField() { + assertUserException(() -> ops.validateNestedStructField(nestedSchema(), ColumnPath.fromDotName("m.key.k"), + "drop"), + "Cannot drop MAP key nested column"); + assertUserException(() -> ops.validateNestedStructField(nestedSchema(), ColumnPath.fromDotName("s.missing"), + "rename"), + "Column path does not exist in Iceberg schema"); + } + private void invokeValidateForModifyColumn(Column column, NestedField currentCol) throws Throwable { invokeValidationMethod(validateForModifyColumnMethod, column, currentCol); } @@ -203,8 +1424,9 @@ private void assertUserException(ThrowingRunnable runnable, String expectedMessa runnable.run(); Assert.fail("expected UserException"); } catch (Throwable t) { - Assert.assertTrue(t instanceof UserException); - Assert.assertTrue(t.getMessage().contains(expectedMessage)); + Assert.assertTrue("expected UserException but got " + t, t instanceof UserException); + Assert.assertTrue("expected message containing '" + expectedMessage + "' but was '" + + t.getMessage() + "'", t.getMessage().contains(expectedMessage)); } } @@ -212,4 +1434,86 @@ private void assertUserException(ThrowingRunnable runnable, String expectedMessa private interface ThrowingRunnable { void run() throws Throwable; } + + private Schema nestedSchema() { + return new Schema( + Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "s", Types.StructType.of( + Types.NestedField.optional(3, "a", Types.IntegerType.get()))), + Types.NestedField.optional(4, "arr", Types.ListType.ofOptional(5, + Types.StructType.of(Types.NestedField.optional(6, "x", Types.IntegerType.get())))), + Types.NestedField.optional(7, "m", Types.MapType.ofOptional(8, 9, + Types.StringType.get(), + Types.StructType.of(Types.NestedField.optional(10, "v", Types.IntegerType.get()))))); + } + + private Schema mixedCaseNestedSchema() { + return new Schema( + Types.NestedField.optional(1, "Id", Types.IntegerType.get()), + Types.NestedField.optional(2, "Info", Types.StructType.of( + Types.NestedField.optional(3, "Metric", Types.IntegerType.get()), + Types.NestedField.optional(4, "Label", Types.StringType.get()))), + Types.NestedField.optional(5, "Events", Types.ListType.ofOptional(6, + Types.StructType.of(Types.NestedField.optional(7, "Score", Types.IntegerType.get())))), + Types.NestedField.optional(8, "Attrs", Types.MapType.ofOptional(9, 10, + Types.StringType.get(), + Types.StructType.of(Types.NestedField.optional(11, "Code", Types.IntegerType.get()))))); + } + + private Schema primitiveContainerSchema() { + return new Schema( + Types.NestedField.optional(1, "arr", + Types.ListType.ofOptional(2, Types.IntegerType.get())), + Types.NestedField.optional(3, "m", Types.MapType.ofOptional( + 4, 5, Types.StringType.get(), Types.IntegerType.get()))); + } + + private Schema mappedPrimitiveSchema() { + return new Schema( + Types.NestedField.required(1, "top_uuid", Types.UUIDType.get()), + Types.NestedField.required(2, "top_other", Types.IntegerType.get()), + Types.NestedField.optional(3, "info", Types.StructType.of( + Types.NestedField.required(4, "uuid_value", Types.UUIDType.get()), + Types.NestedField.required(5, "tz_value", Types.TimestampType.withZone()), + Types.NestedField.required(6, "other", Types.IntegerType.get())))); + } + + private Schema mappedComplexSchema() { + return new Schema(Types.NestedField.optional(1, "outer", Types.StructType.of( + Types.NestedField.optional(2, "payload", Types.StructType.of( + Types.NestedField.optional(3, "uuid_value", Types.UUIDType.get()), + Types.NestedField.optional(4, "binary_value", Types.BinaryType.get()), + Types.NestedField.optional(5, "fixed_value", Types.FixedType.ofLength(8)), + Types.NestedField.optional(6, "tz_value", Types.TimestampType.withZone()), + Types.NestedField.optional(7, "metric", Types.IntegerType.get()), + Types.NestedField.optional(8, "events", + Types.ListType.ofOptional(9, Types.UUIDType.get())), + Types.NestedField.optional(10, "attrs", Types.MapType.ofOptional( + 11, 12, Types.FixedType.ofLength(4), Types.TimestampType.withZone()))))))); + } + + private StructType mappedPayloadDorisType(Type metricType, int fixedLength, Type mapKeyType) { + return new StructType( + new StructField("uuid_value", ScalarType.createVarbinaryType(16)), + new StructField("binary_value", + ScalarType.createVarbinaryType(ScalarType.MAX_VARBINARY_LENGTH)), + new StructField("fixed_value", ScalarType.createVarbinaryType(fixedLength)), + new StructField("tz_value", ScalarType.createTimeStampTzType(6)), + new StructField("metric", metricType), + new StructField("events", ArrayType.create( + ScalarType.createVarbinaryType(16), true)), + new StructField("attrs", new MapType( + mapKeyType, ScalarType.createTimeStampTzType(6)))); + } + + private Schema requiredNestedSchema() { + return new Schema(Types.NestedField.required(1, "info", Types.StructType.of( + Types.NestedField.required(2, "metric", Types.IntegerType.get()), + Types.NestedField.required(3, "child", Types.StructType.of( + Types.NestedField.required(4, "value", Types.IntegerType.get()))), + Types.NestedField.required(5, "events", Types.ListType.ofRequired( + 6, Types.IntegerType.get())), + Types.NestedField.required(7, "attrs", Types.MapType.ofRequired( + 8, 9, Types.StringType.get(), Types.IntegerType.get()))))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index ee34b4bee50b0c..f29ccd6182bd9f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -78,6 +78,28 @@ import java.util.UUID; public class IcebergUtilsTest { + @Test + public void testGetFileFormatUsesPropertiesWithoutPlanningDataFiles() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(table.currentSnapshot()).thenReturn(Mockito.mock(Snapshot.class)); + + Assert.assertEquals(org.apache.iceberg.FileFormat.PARQUET, IcebergUtils.getFileFormat(table)); + // Do not call newScan planFiles() + Mockito.verify(table, Mockito.never()).newScan(); + } + + @Test + public void testGetFileFormatUsesConfiguredTableFormat() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.properties()).thenReturn( + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, "orc")); + + Assert.assertEquals(org.apache.iceberg.FileFormat.ORC, IcebergUtils.getFileFormat(table)); + // Do not call newScan planFiles() + Mockito.verify(table, Mockito.never()).newScan(); + } + @Test public void testGetIcebergViewUsesSessionCatalogWithDelegatedCredential() { ConnectContext context = new ConnectContext(); @@ -342,6 +364,20 @@ public void testParseSchemaPreservesInitialDefault() { Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); } + @Test + public void testParseSchemaPreservesNestedNonBinaryInitialDefault() { + Schema schema = new Schema(Types.NestedField.optional(10, "s", Types.StructType.of( + Types.NestedField.optional("added_int") + .withId(11) + .ofType(Types.IntegerType.get()) + .withInitialDefault(7) + .build()))); + + List columns = IcebergUtils.parseSchema(schema, true, false); + + Assert.assertEquals("7", columns.get(0).getChildren().get(0).getDefaultValue()); + } + @Test public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/ReauthenticatingRestSessionCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/ReauthenticatingRestSessionCatalogTest.java new file mode 100644 index 00000000000000..2132a38febb656 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/ReauthenticatingRestSessionCatalogTest.java @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SessionCatalog.SessionContext; +import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.iceberg.rest.RESTSessionCatalog; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +public class ReauthenticatingRestSessionCatalogTest { + + private static final Namespace NS = Namespace.empty(); + + /** + * A stand-in for a live REST catalog: throws the configured failure on every namespace listing until the + * failure is cleared, and records whether it was closed. All overridden methods avoid the real + * RESTSessionCatalog internals, so no initialization or network is involved. + */ + private static final class FakeRestSessionCatalog extends RESTSessionCatalog { + private final String label; + private final RuntimeException failure; + private final AtomicInteger listCalls = new AtomicInteger(); + private volatile boolean closed; + + FakeRestSessionCatalog(String label, RuntimeException failure) { + this.label = label; + this.failure = failure; + } + + @Override + public String name() { + return label; + } + + @Override + public List listNamespaces(SessionContext context, Namespace ns) { + listCalls.incrementAndGet(); + if (failure != null) { + throw failure; + } + return Collections.singletonList(Namespace.of(label)); + } + + @Override + public void close() { + closed = true; + } + } + + private static NotAuthorizedException notAuthorized() { + return new NotAuthorizedException("Not authorized: %s", "the token expired"); + } + + private static SessionContext delegatedUserContext() { + return new SessionContext(UUID.randomUUID().toString(), "alice", + Collections.singletonMap("token", "user-token"), Collections.emptyMap()); + } + + @Test + public void testNotAuthorizedRebuildsClientAndRetriesOnce() { + FakeRestSessionCatalog wedged = new FakeRestSessionCatalog("wedged", notAuthorized()); + FakeRestSessionCatalog fresh = new FakeRestSessionCatalog("fresh", null); + AtomicInteger rebuilds = new AtomicInteger(); + ReauthenticatingRestSessionCatalog catalog = new ReauthenticatingRestSessionCatalog(wedged, () -> { + rebuilds.incrementAndGet(); + return fresh; + }); + + List namespaces = catalog.listNamespaces(SessionContext.createEmpty(), NS); + + Assertions.assertEquals(Collections.singletonList(Namespace.of("fresh")), namespaces); + Assertions.assertEquals(1, rebuilds.get()); + Assertions.assertEquals(1, wedged.listCalls.get()); + Assertions.assertEquals(1, fresh.listCalls.get()); + Assertions.assertTrue(wedged.closed, "the wedged client must be closed after replacement"); + Assertions.assertSame(fresh, catalog.currentDelegate()); + } + + @Test + public void testNotAuthorizedWrappedInAnotherExceptionIsDetected() { + FakeRestSessionCatalog wedged = new FakeRestSessionCatalog("wedged", + new RuntimeException("Failed to list database names", notAuthorized())); + FakeRestSessionCatalog fresh = new FakeRestSessionCatalog("fresh", null); + ReauthenticatingRestSessionCatalog catalog = + new ReauthenticatingRestSessionCatalog(wedged, () -> fresh); + + List namespaces = catalog.listNamespaces(SessionContext.createEmpty(), NS); + + Assertions.assertEquals(Collections.singletonList(Namespace.of("fresh")), namespaces); + } + + @Test + public void testStillNotAuthorizedAfterRebuildPropagatesWithoutLooping() { + FakeRestSessionCatalog wedged = new FakeRestSessionCatalog("wedged", notAuthorized()); + FakeRestSessionCatalog stillWedged = new FakeRestSessionCatalog("still-wedged", notAuthorized()); + AtomicInteger rebuilds = new AtomicInteger(); + ReauthenticatingRestSessionCatalog catalog = new ReauthenticatingRestSessionCatalog(wedged, () -> { + rebuilds.incrementAndGet(); + return stillWedged; + }); + + Assertions.assertThrows(NotAuthorizedException.class, + () -> catalog.listNamespaces(SessionContext.createEmpty(), NS)); + Assertions.assertEquals(1, rebuilds.get(), "exactly one rebuild, no retry loop"); + Assertions.assertEquals(1, wedged.listCalls.get()); + Assertions.assertEquals(1, stillWedged.listCalls.get()); + } + + @Test + public void testNonAuthFailuresAreNotRetried() { + FakeRestSessionCatalog failing = new FakeRestSessionCatalog("failing", + new RuntimeException("connection reset")); + AtomicInteger rebuilds = new AtomicInteger(); + ReauthenticatingRestSessionCatalog catalog = new ReauthenticatingRestSessionCatalog(failing, () -> { + rebuilds.incrementAndGet(); + return new FakeRestSessionCatalog("fresh", null); + }); + + Assertions.assertThrows(RuntimeException.class, + () -> catalog.listNamespaces(SessionContext.createEmpty(), NS)); + Assertions.assertEquals(0, rebuilds.get()); + Assertions.assertEquals(1, failing.listCalls.get()); + Assertions.assertFalse(failing.closed); + } + + @Test + public void testDelegatedUserSessionIsNotRecovered() { + // A 401 for a request carrying a per-user delegated credential means that user's token is invalid. + // Rebuilding the shared client cannot fix it and must not be triggered by it. + FakeRestSessionCatalog wedged = new FakeRestSessionCatalog("wedged", notAuthorized()); + AtomicInteger rebuilds = new AtomicInteger(); + ReauthenticatingRestSessionCatalog catalog = new ReauthenticatingRestSessionCatalog(wedged, () -> { + rebuilds.incrementAndGet(); + return new FakeRestSessionCatalog("fresh", null); + }); + + Assertions.assertThrows(NotAuthorizedException.class, + () -> catalog.listNamespaces(delegatedUserContext(), NS)); + Assertions.assertEquals(0, rebuilds.get()); + Assertions.assertFalse(wedged.closed); + } + + @Test + public void testAsCatalogViewRoutesThroughRecovery() { + // The default Catalog handed to IcebergMetadataOps is asCatalog(empty); it must inherit the same + // recovery because it calls back into this session catalog. + FakeRestSessionCatalog wedged = new FakeRestSessionCatalog("wedged", notAuthorized()); + FakeRestSessionCatalog fresh = new FakeRestSessionCatalog("fresh", null); + ReauthenticatingRestSessionCatalog catalog = + new ReauthenticatingRestSessionCatalog(wedged, () -> fresh); + + List namespaces = ((org.apache.iceberg.catalog.SupportsNamespaces) + catalog.asCatalog(SessionContext.createEmpty())).listNamespaces(NS); + + Assertions.assertEquals(Collections.singletonList(Namespace.of("fresh")), namespaces); + Assertions.assertTrue(wedged.closed); + } + + @Test + public void testCloseClosesCurrentDelegate() throws Exception { + FakeRestSessionCatalog delegate = new FakeRestSessionCatalog("delegate", null); + ReauthenticatingRestSessionCatalog catalog = + new ReauthenticatingRestSessionCatalog(delegate, () -> delegate); + + catalog.close(); + + Assertions.assertTrue(delegate.closed); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java index 77d318518f326e..39fc15ddbe1a7e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelperTest.java @@ -18,19 +18,34 @@ package org.apache.doris.datasource.iceberg.helper; import org.apache.doris.thrift.TFileContent; +import org.apache.doris.thrift.TIcebergColumnStats; import org.apache.doris.thrift.TIcebergCommitData; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; +import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Test for IcebergWriterHelper DeleteFile conversion @@ -58,6 +73,242 @@ public void setUp() { } + @Test + public void testConvertToWriterResultRespectsNoneMetricsMode() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(Map.of(2, 128L)); + columnStats.setValueCounts(Map.of(2, 10L)); + columnStats.setNullValueCounts(Map.of(2, 0L)); + columnStats.setLowerBounds(Map.of(2, ByteBuffer.wrap(new byte[] {0x01}))); + columnStats.setUpperBounds(Map.of(2, ByteBuffer.wrap(new byte[] {0x02}))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + WriteResult result = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)); + DataFile dataFile = result.dataFiles()[0]; + + Assertions.assertTrue(dataFile.columnSizes() == null || dataFile.columnSizes().isEmpty()); + Assertions.assertTrue(dataFile.valueCounts() == null || dataFile.valueCounts().isEmpty()); + Assertions.assertTrue(dataFile.nullValueCounts() == null || dataFile.nullValueCounts().isEmpty()); + Assertions.assertTrue(dataFile.lowerBounds() == null || dataFile.lowerBounds().isEmpty()); + Assertions.assertTrue(dataFile.upperBounds() == null || dataFile.upperBounds().isEmpty()); + } + + @Test + public void testConvertToWriterResultCountsModeOmitsBounds() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(Map.of(2, 128L)); + columnStats.setValueCounts(Map.of(2, 10L)); + columnStats.setNullValueCounts(Map.of(2, 0L)); + columnStats.setLowerBounds(Map.of( + 2, Conversions.toByteBuffer(Types.StringType.get(), "abcdefgh"))); + columnStats.setUpperBounds(Map.of( + 2, Conversions.toByteBuffer(Types.StringType.get(), "ijklmnop"))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + Assertions.assertEquals(Map.of(2, 128L), dataFile.columnSizes()); + Assertions.assertEquals(Map.of(2, 10L), dataFile.valueCounts()); + Assertions.assertEquals(Map.of(2, 0L), dataFile.nullValueCounts()); + Assertions.assertTrue(dataFile.lowerBounds() == null || dataFile.lowerBounds().isEmpty()); + Assertions.assertTrue(dataFile.upperBounds() == null || dataFile.upperBounds().isEmpty()); + } + + @Test + public void testConvertToWriterResultTruncatesStringAndBinaryBounds() { + Schema boundsSchema = new Schema( + Types.NestedField.optional(1, "text", Types.StringType.get()), + Types.NestedField.optional(2, "payload", Types.BinaryType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(boundsSchema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(3)")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setLowerBounds(Map.of( + 1, Conversions.toByteBuffer(Types.StringType.get(), "abcdef"), + 2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4}))); + columnStats.setUpperBounds(Map.of( + 1, Conversions.toByteBuffer(Types.StringType.get(), "uvwxyz"), + 2, ByteBuffer.wrap(new byte[] {1, 2, 3, 4}))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + Assertions.assertEquals("abc", Conversions.fromByteBuffer( + Types.StringType.get(), dataFile.lowerBounds().get(1)).toString()); + Assertions.assertEquals("uvx", Conversions.fromByteBuffer( + Types.StringType.get(), dataFile.upperBounds().get(1)).toString()); + Assertions.assertEquals(ByteBuffer.wrap(new byte[] {1, 2, 3}), dataFile.lowerBounds().get(2)); + Assertions.assertEquals(ByteBuffer.wrap(new byte[] {1, 2, 4}), dataFile.upperBounds().get(2)); + } + + @Test + public void testConvertToWriterResultPreservesOrcUpperBoundWithoutTruncatedSuccessor() { + Schema boundsSchema = new Schema( + Types.NestedField.optional(1, "text", Types.StringType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(boundsSchema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "orc", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(1)")); + + String maxWithoutSuccessor = new String(Character.toChars(Character.MAX_CODE_POINT)) + "tail"; + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setUpperBounds(Map.of( + 1, Conversions.toByteBuffer(Types.StringType.get(), maxWithoutSuccessor))); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.orc"); + commitData.setRowCount(1); + commitData.setFileSize(128); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + Assertions.assertNotNull(dataFile.upperBounds()); + Assertions.assertEquals(maxWithoutSuccessor, Conversions.fromByteBuffer( + Types.StringType.get(), dataFile.upperBounds().get(1)).toString()); + } + + @Test + public void testConvertToWriterResultBuildsMetricsPolicyOncePerBatch() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); + + TIcebergCommitData firstCommit = new TIcebergCommitData(); + firstCommit.setFilePath("/path/to/first.parquet"); + firstCommit.setRowCount(10); + firstCommit.setFileSize(1024); + + TIcebergCommitData secondCommit = new TIcebergCommitData(); + secondCommit.setFilePath("/path/to/second.parquet"); + secondCommit.setRowCount(20); + secondCommit.setFileSize(2048); + + IcebergWriterHelper.convertToWriterResult(table, List.of(firstCommit, secondCommit)); + + // One schema lookup is made by Iceberg's policy builder and one is captured for all files in the batch. + Mockito.verify(table, Mockito.times(2)).schema(); + } + + @Test + public void testConvertToWriterResultHandlesV3TransactionTableLineageMetrics(@TempDir Path tempDir) { + HadoopTables tables = new HadoopTables(new Configuration()); + Table baseTable = tables.create(schema, unpartitionedSpec, SortOrder.unsorted(), Map.of( + TableProperties.FORMAT_VERSION, "3", + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "truncate(16)"), + tempDir.resolve("table").toUri().toString()); + Table transactionTable = baseTable.newTransaction().table(); + + int rowId = MetadataColumns.ROW_ID.fieldId(); + int sequenceNumberId = MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(); + ByteBuffer rowIdBound = Conversions.toByteBuffer(MetadataColumns.ROW_ID.type(), 7L); + ByteBuffer sequenceNumberBound = Conversions.toByteBuffer( + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.type(), 3L); + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setLowerBounds(Map.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); + columnStats.setUpperBounds(Map.of(rowId, rowIdBound, sequenceNumberId, sequenceNumberBound)); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/v3-data.parquet"); + commitData.setRowCount(1); + commitData.setFileSize(128); + commitData.setColumnStats(columnStats); + + DataFile dataFile = Assertions.assertDoesNotThrow( + () -> IcebergWriterHelper.convertToWriterResult( + transactionTable, List.of(commitData)).dataFiles()[0]); + Assertions.assertEquals(rowIdBound, dataFile.lowerBounds().get(rowId)); + Assertions.assertEquals(rowIdBound, dataFile.upperBounds().get(rowId)); + Assertions.assertEquals(sequenceNumberBound, dataFile.lowerBounds().get(sequenceNumberId)); + Assertions.assertEquals(sequenceNumberBound, dataFile.upperBounds().get(sequenceNumberId)); + } + + @Test + public void testConvertToWriterResultSuppressesLogicalMetricsBelowRepeatedFields() { + Schema repeatedSchema = new Schema( + Types.NestedField.optional(1, "items", + Types.ListType.ofOptional(2, Types.IntegerType.get())), + Types.NestedField.optional(3, "attributes", + Types.MapType.ofOptional(4, 5, Types.StringType.get(), Types.StringType.get())), + Types.NestedField.optional(6, "top_level", Types.IntegerType.get())); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schema()).thenReturn(repeatedSchema); + Mockito.when(table.spec()).thenReturn(unpartitionedSpec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "full")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(Map.of(2, 20L, 4, 40L, 5, 50L, 6, 60L)); + columnStats.setValueCounts(Map.of(2, 2L, 4, 4L, 5, 5L, 6, 6L)); + columnStats.setNullValueCounts(Map.of(2, 0L, 4, 0L, 5, 0L, 6, 0L)); + columnStats.setLowerBounds(Map.of( + 2, Conversions.toByteBuffer(Types.IntegerType.get(), 2), + 4, Conversions.toByteBuffer(Types.StringType.get(), "key"), + 5, Conversions.toByteBuffer(Types.StringType.get(), "value"), + 6, Conversions.toByteBuffer(Types.IntegerType.get(), 6))); + columnStats.setUpperBounds(columnStats.getLowerBounds()); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/repeated.parquet"); + commitData.setRowCount(6); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + Assertions.assertEquals(Map.of(2, 20L, 4, 40L, 5, 50L, 6, 60L), dataFile.columnSizes()); + Assertions.assertEquals(Map.of(6, 6L), dataFile.valueCounts()); + Assertions.assertEquals(Map.of(6, 0L), dataFile.nullValueCounts()); + Assertions.assertEquals(Map.of(6, columnStats.getLowerBounds().get(6)), dataFile.lowerBounds()); + Assertions.assertEquals(Map.of(6, columnStats.getUpperBounds().get(6)), dataFile.upperBounds()); + } + @Test public void testConvertToDeleteFiles_EmptyList() { List commitDataList = new ArrayList<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java new file mode 100644 index 00000000000000..6a714107a6e856 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergDeleteFileFilterTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg.source; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class IcebergDeleteFileFilterTest { + @Test + public void testValidateDeletionVectorMetadataAcceptsLongRange() { + Assertions.assertDoesNotThrow(() -> IcebergDeleteFileFilter.validateDeletionVectorMetadata( + "puffin.dv", 1L << 40, (1L << 32) + 17, (1L << 30) + 19)); + } + + @Test + public void testValidateDeletionVectorMetadataRejectsInvalidRange() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, null, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, 1L, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", -1, 1L, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, -1L, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, 1L, -1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata( + "puffin.dv", Long.MAX_VALUE, Long.MAX_VALUE, 1L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> IcebergDeleteFileFilter.validateDeletionVectorMetadata("puffin.dv", 100, 90L, 11L)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 2a331f57462152..06d7b68c50d93c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -17,27 +17,51 @@ package org.apache.doris.datasource.iceberg.source; +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.TableFormatType; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergMvccSnapshot; +import org.apache.doris.datasource.iceberg.IcebergPartitionInfo; +import org.apache.doris.datasource.iceberg.IcebergSnapshot; +import org.apache.doris.datasource.iceberg.IcebergSnapshotCacheValue; +import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.mvcc.MvccTableInfo; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; +import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.TIcebergDeleteFileDesc; +import org.apache.doris.thrift.TPushAggOp; import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PositionDeletesScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ScanTaskUtil; import org.junit.Assert; @@ -52,11 +76,20 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; public class IcebergScanNodeTest { private static final long MB = 1024L * 1024L; + @SuppressWarnings("unchecked") + private static Optional>> extractNameMapping( + IcebergScanNode node) throws Exception { + Method method = IcebergScanNode.class.getDeclaredMethod("extractNameMapping"); + method.setAccessible(true); + return (Optional>>) method.invoke(node); + } + private static class TestIcebergScanNode extends IcebergScanNode { private final boolean enableMappingVarbinary; private TableScan tableScan; @@ -88,10 +121,208 @@ public boolean isBatchMode() { protected boolean getEnableMappingVarbinary() { return enableMappingVarbinary; } + + @Override + public List getPathPartitionKeys() { + return Collections.emptyList(); + } + + void addSlot(int slotId, Column column) { + SlotDescriptor slot = new SlotDescriptor(new SlotId(slotId), desc.getId()); + slot.setColumn(column); + desc.addSlot(slot); + } + + int enableAndGetIcebergScanSemanticsVersion() { + params = new TFileScanRangeParams(); + enableCurrentIcebergScanSemantics(); + return params.getIcebergScanSemanticsVersion(); + } + } + + @Test + public void testEmitsCurrentIcebergScanSemanticsCapability() { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + + Assert.assertEquals(IcebergScanNode.ICEBERG_SCAN_SEMANTICS_VERSION, + node.enableAndGetIcebergScanSemanticsVersion()); + } + + @Test + public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Table table = Mockito.mock(Table.class); + setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(IcebergExternalTable.class)); + setIcebergSource(node, source); + + Mockito.when(table.properties()).thenReturn(Collections.emptyMap()); + Assert.assertFalse(extractNameMapping(node).isPresent()); + + Mockito.when(table.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + Optional>> emptyMapping = extractNameMapping(node); + Assert.assertTrue(emptyMapping.isPresent()); + Assert.assertTrue(emptyMapping.get().isEmpty()); + } + + @Test + public void testSnapshotCacheIgnoresIdlessNameMappingWrapper() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.properties()).thenReturn(Collections.singletonMap( + TableProperties.DEFAULT_NAME_MAPPING, + "[{\"names\":[\"legacy_wrapper\"],\"fields\":[" + + "{\"field-id\":7,\"names\":[\"legacy_child\"]}]}]")); + + IcebergSnapshotCacheValue snapshotCacheValue = new IcebergSnapshotCacheValue( + new IcebergPartitionInfo(Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap()), + new IcebergSnapshot(1L, 1L), + IcebergUtils.getNameMapping(table)); + + Assert.assertTrue(snapshotCacheValue.getNameMapping().isPresent()); + Assert.assertEquals(Collections.singletonList("legacy_child"), + snapshotCacheValue.getNameMapping().get().get(7)); + Assert.assertEquals(Collections.singleton(7), + snapshotCacheValue.getNameMapping().get().keySet()); + } + + @Test + public void testExtractNameMappingUsesStatementPinnedMetadataAfterPropertyRefresh() throws Exception { + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.properties()).thenReturn( + Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING, "[]")); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, 11L), Optional.empty()))); + try { + Assert.assertFalse(extractNameMapping(node).isPresent()); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void testSystemTableProjectionMatchesFileSlotOrder() throws Exception { + Schema systemTableSchema = new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()), + Types.NestedField.required(2, "record_count", Types.LongType.get()), + Types.NestedField.optional(3, "readable_metrics", Types.StructType.of( + Types.NestedField.optional(4, "id", Types.StructType.of( + Types.NestedField.optional(5, "lower_bound", Types.IntegerType.get())))))); + Table systemTable = Mockito.mock(Table.class); + Mockito.when(systemTable.schema()).thenReturn(systemTableSchema); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, systemTable); + node.addSlot(1, new Column("RECORD_COUNT", Type.BIGINT)); + node.addSlot(2, new Column(Column.GLOBAL_ROWID_COL + "system_table", Type.BIGINT)); + node.addSlot(3, new Column("FILE_PATH", Type.STRING)); + + List filters = Collections.singletonList(Expressions.greaterThan("record_count", 0L)); + Schema projectedSchema = node.getSystemTableProjectedSchema(filters, false); + + Assert.assertEquals(2, projectedSchema.columns().size()); + Assert.assertEquals("record_count", projectedSchema.columns().get(0).name()); + Assert.assertEquals("file_path", projectedSchema.columns().get(1).name()); + Assert.assertNull(projectedSchema.findField("readable_metrics")); + } + + @Test + public void testSystemTableProjectionRejectsUnmaterializedFilterColumn() throws Exception { + Schema systemTableSchema = new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()), + Types.NestedField.required(2, "record_count", Types.LongType.get())); + Table systemTable = Mockito.mock(Table.class); + Mockito.when(systemTable.schema()).thenReturn(systemTableSchema); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, systemTable); + node.addSlot(1, new Column("record_count", Type.BIGINT)); + + try { + node.getSystemTableProjectedSchema( + Collections.singletonList(Expressions.equal("file_path", "data.parquet")), true); + Assert.fail("Filter columns must be materialized by the planner"); + } catch (UserException e) { + Assert.assertTrue(e.getMessage().contains("filter column file_path is not materialized")); + } + } + + private static class CountPlanningIcebergScanNode extends IcebergScanNode { + private final TableScan tableScan; + private final long snapshotCount; + private int snapshotCountCalls; + + CountPlanningIcebergScanNode(SessionVariable sv, TableScan tableScan, long snapshotCount) { + super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), sv, ScanContext.EMPTY); + this.tableScan = tableScan; + this.snapshotCount = snapshotCount; + } + + @Override + public TableScan createTableScan() { + return tableScan; + } + + @Override + public long getCountFromSnapshot() { + ++snapshotCountCalls; + return snapshotCount; + } } @Test - public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { + public void testTableLevelCountSplitPlanningRequiresCountStar() { + SessionVariable sv = Mockito.mock(SessionVariable.class); + Mockito.when(sv.getEnableExternalTableBatchMode()).thenReturn(false); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(tableScan.snapshot()).thenReturn(Mockito.mock(Snapshot.class)); + + // COUNT(required_col) carries a non-empty semantic argument list. Even though its result + // equals COUNT(*) for valid data, BE intentionally reads the column to enforce schema + // contracts. FE must therefore leave all real file tasks available to that fallback. + CountPlanningIcebergScanNode countColumnNode = + new CountPlanningIcebergScanNode(sv, tableScan, 30_000); + countColumnNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countColumnNode.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7))); + Assert.assertFalse(countColumnNode.isBatchMode()); + Assert.assertEquals(0, countColumnNode.snapshotCountCalls); + + // COUNT(*) has an explicitly empty argument list, so snapshot row count remains eligible + // and doGetSplits may retain only representative tasks for parallel materialization. + CountPlanningIcebergScanNode countStarNode = + new CountPlanningIcebergScanNode(sv, tableScan, 30_000); + countStarNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countStarNode.setPushDownCountSlotIds(Collections.emptyList()); + Assert.assertFalse(countStarNode.isBatchMode()); + Assert.assertEquals(1, countStarNode.snapshotCountCalls); + } + + @Test + public void testInitialDefaultMetadataUsesCurrentSchemaForOrdinaryScan() throws Exception { Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") .withId(7) .ofType(Types.BinaryType.get()) @@ -106,6 +337,7 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { Mockito.when(snapshot.schemaId()).thenReturn(11); Table table = Mockito.mock(Table.class); Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + Mockito.when(table.schema()).thenReturn(currentSchema); TableScan snapshotScan = Mockito.mock(TableScan.class); Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); Mockito.when(snapshotScan.table()).thenReturn(table); @@ -113,11 +345,148 @@ public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); node.setTableScan(snapshotScan); + setIcebergTable(node, table); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(Mockito.mock(TableIf.class)); + setIcebergSource(node, source); Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertTrue(defaults.isEmpty()); + } + + @Test + public void testInitialDefaultMetadataUsesStatementPinnedSchemaAfterCacheInvalidation() throws Exception { + Schema pinnedSchema = new Schema(11, List.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Schema refreshedSchema = new Schema(12, + List.of(Types.NestedField.optional(8, "replacement", Types.IntegerType.get()))); + Table refreshedTable = Mockito.mock(Table.class); + Mockito.when(refreshedTable.schema()).thenReturn(refreshedSchema); + Mockito.when(refreshedTable.schemas()).thenReturn(Map.of( + pinnedSchema.schemaId(), pinnedSchema, + refreshedSchema.schemaId(), refreshedSchema)); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + setIcebergTable(node, refreshedTable); + setIcebergSource(node, source); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, pinnedSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void testInitialDefaultMetadataUsesSnapshotSchemaForExplicitSelection() throws Exception { + Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Mockito.when(snapshot.schemaId()).thenReturn(11); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + TableScan snapshotScan = Mockito.mock(TableScan.class); + Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); + Mockito.when(snapshotScan.table()).thenReturn(table); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + IcebergTableQueryInfo selectedSnapshot = Mockito.mock(IcebergTableQueryInfo.class); + Mockito.when(selectedSnapshot.getSchemaId()).thenReturn(11); + + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(snapshotScan); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(selectedSnapshot).when(node).getSpecifiedSnapshot(); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); } + @Test + public void testInitialDefaultMetadataUsesStatementPinnedBranchSchema() throws Exception { + Schema dataSnapshotSchema = new Schema(11, List.of(Types.NestedField.optional("string_default") + .withId(7) + .ofType(Types.StringType.get()) + .withInitialDefault("not-base64") + .build())); + Schema branchSchema = new Schema(12, List.of(Types.NestedField.optional("binary_default") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build())); + Snapshot dataSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(dataSnapshot.schemaId()).thenReturn(dataSnapshotSchema.schemaId()); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Map.of( + dataSnapshotSchema.schemaId(), dataSnapshotSchema, + branchSchema.schemaId(), branchSchema)); + TableScan branchScan = Mockito.mock(TableScan.class); + Mockito.when(branchScan.snapshot()).thenReturn(dataSnapshot); + Mockito.when(branchScan.table()).thenReturn(table); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(targetTable.getName()).thenReturn("tbl"); + Mockito.when(targetTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("catalog"); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.setTableScan(branchScan); + setIcebergTable(node, table); + setIcebergSource(node, source); + Mockito.doReturn(new IcebergTableQueryInfo(1L, "branch", branchSchema.schemaId())) + .when(node).getSpecifiedSnapshot(); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext(); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + statementContext.setSnapshot(new MvccTableInfo(targetTable), new IcebergMvccSnapshot( + new IcebergSnapshotCacheValue(new IcebergPartitionInfo( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + new IcebergSnapshot(1L, branchSchema.schemaId())))); + try { + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), + node.getBase64EncodedInitialDefaultsForScan()); + } finally { + ConnectContext.remove(); + } + } + @Test public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() throws Exception { Schema systemTableSchema = new Schema(Types.NestedField.optional("binary_default") @@ -142,6 +511,18 @@ public void testInitialDefaultMetadataUsesSystemTableSchemaWithoutTableScan() th Mockito.verify(node, Mockito.never()).createTableScan(); } + private static void setIcebergTable(IcebergScanNode node, Table table) throws Exception { + Field icebergTableField = IcebergScanNode.class.getDeclaredField("icebergTable"); + icebergTableField.setAccessible(true); + icebergTableField.set(node, table); + } + + private static void setIcebergSource(IcebergScanNode node, IcebergSource source) throws Exception { + Field sourceField = IcebergScanNode.class.getDeclaredField("source"); + sourceField.setAccessible(true); + sourceField.set(node, source); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); @@ -180,6 +561,7 @@ public void testSetIcebergParamsKeepsDeletionVectorOffsetAsLong() throws Excepti IcebergSplit split = new IcebergSplit(LocationPath.of(dataPath), 0, 128, 128, new String[0], 3, Collections.emptyMap(), new ArrayList<>(), dataPath); split.setTableFormatType(TableFormatType.ICEBERG); + split.setSplitFileFormat(FileFormat.PARQUET); split.setFirstRowId(10L); split.setLastUpdatedSequenceNumber(20L); split.setDeleteFileFilters(Collections.emptyList(), Collections.singletonList( @@ -201,6 +583,53 @@ public void testSetIcebergParamsKeepsDeletionVectorOffsetAsLong() throws Excepti Assert.assertEquals((long) Integer.MAX_VALUE + 7L, deleteFileDesc.getContentSizeInBytes()); } + @Test + public void testSetIcebergParamsUsesSplitFileFormat() throws Exception { + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + String dataPath = "file:///tmp/data-file.orc"; + IcebergSplit split = new IcebergSplit(LocationPath.of(dataPath), 0, 128, 128, new String[0], + 2, Collections.emptyMap(), new ArrayList<>(), dataPath); + split.setTableFormatType(TableFormatType.ICEBERG); + split.setSplitFileFormat(FileFormat.ORC); + + Method method = IcebergScanNode.class.getDeclaredMethod("setIcebergParams", + TFileRangeDesc.class, IcebergSplit.class); + method.setAccessible(true); + + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + method.invoke(node, rangeDesc, split); + + Assert.assertEquals(TFileFormatType.FORMAT_ORC, rangeDesc.getFormatType()); + } + + @Test + public void testPositionDeleteSystemTableValidatesDeletionVectorMetadata() throws Exception { + DeleteFile deleteFile = Mockito.mock(DeleteFile.class); + Mockito.when(deleteFile.path()).thenReturn("file:///tmp/delete-shared.puffin"); + Mockito.when(deleteFile.format()).thenReturn(FileFormat.PUFFIN); + Mockito.when(deleteFile.fileSizeInBytes()).thenReturn(100L); + Mockito.when(deleteFile.contentOffset()).thenReturn(null); + Mockito.when(deleteFile.contentSizeInBytes()).thenReturn(10L); + + PositionDeletesScanTask task = Mockito.mock(PositionDeletesScanTask.class); + Mockito.when(task.file()).thenReturn(deleteFile); + Mockito.when(task.start()).thenReturn(0L); + Mockito.when(task.length()).thenReturn(100L); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + Method method = IcebergScanNode.class.getDeclaredMethod( + "createIcebergPositionDeleteSysSplit", PositionDeletesScanTask.class); + method.setAccessible(true); + + try { + method.invoke(node, task); + Assert.fail("position_deletes planning should reject invalid deletion vector metadata"); + } catch (InvocationTargetException e) { + Assert.assertTrue(e.getCause() instanceof IllegalArgumentException); + Assert.assertTrue(e.getCause().getMessage().contains("delete-shared.puffin")); + } + } + @Test public void testSetIcebergParamsPropagatesPositionDeleteFileFormat() throws Exception { SessionVariable sv = new SessionVariable(); @@ -215,6 +644,7 @@ public void testSetIcebergParamsPropagatesPositionDeleteFileFormat() throws Exce IcebergSplit split = new IcebergSplit(LocationPath.of(dataPath), 0, 128, 128, new String[0], 2, Collections.emptyMap(), new ArrayList<>(), dataPath); split.setTableFormatType(TableFormatType.ICEBERG); + split.setSplitFileFormat(FileFormat.PARQUET); split.setDeleteFileFilters(Collections.emptyList(), Collections.singletonList( new IcebergDeleteFileFilter.PositionDelete(deletePath, -1L, -1L, 256L, org.apache.iceberg.FileFormat.ORC))); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java index 99e4aa62dd574d..a76319d108f7cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java @@ -19,11 +19,36 @@ import org.junit.Assert; import org.junit.Test; +import org.mockito.Answers; +import org.mockito.Mockito; import java.lang.reflect.Method; +import java.sql.DatabaseMetaData; public class JdbcClickHouseClientTest { + @Test + public void testDatabaseTermFollowsDriverMetadata() throws Exception { + DatabaseMetaData databaseMetaData = Mockito.mock(DatabaseMetaData.class); + + Mockito.when(databaseMetaData.supportsCatalogsInDataManipulation()).thenReturn(false); + Assert.assertFalse(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData, "0.9.8")); + + Mockito.when(databaseMetaData.supportsCatalogsInDataManipulation()).thenReturn(true); + Assert.assertTrue(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData, "0.7.1")); + + Assert.assertFalse(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData, "0.4.2")); + } + + @Test + public void testClickHouseSpecificTableTypesAreVisible() { + JdbcClickHouseClient client = Mockito.mock(JdbcClickHouseClient.class, Answers.CALLS_REAL_METHODS); + + Assert.assertArrayEquals( + new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE", "MATERIALIZED VIEW"}, + client.getTableTypes()); + } + @Test public void testIsNewClickHouseDriver() { try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 050e547816d404..9200f4319adaf2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.paimon; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.Type; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TFieldPtr; @@ -26,6 +28,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryRowWriter; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.Table; import org.apache.paimon.types.CharType; @@ -39,6 +42,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -114,6 +118,125 @@ public void testGetPartitionInfoMapPreservesNonLowercaseKeys() { Assert.assertEquals("2026-05-26", partitionInfoMap.get("Dt")); } + @Test + public void testGeneratePartitionInfoWithSpecialCharacters() { + List partitionColumns = Arrays.asList( + new Column("source", Type.STRING), + new Column("part_str", Type.STRING), + new Column("pass", Type.STRING)); + Map spec = new LinkedHashMap<>(); + spec.put("source", "dataset/team-a/segment-01"); + spec.put("part_str", "/ymd=20260701/hour=[0-9][0-9]/*.jsonl"); + spec.put("pass", "s1"); + Partition partition = new Partition(spec, 1L, 1L, 1L, 1L, false); + + PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( + partitionColumns, Collections.singletonList(partition), false); + + Assert.assertFalse(partitionInfo.isPartitionInvalid()); + Assert.assertEquals(1, partitionInfo.getNameToPartition().size()); + Assert.assertEquals(1, partitionInfo.getNameToPartitionItem().size()); + String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" + + "/part_str=%2Fymd%3D20260701%2Fhour%3D%5B0-9%5D%5B0-9%5D%2F%2A.jsonl/pass=s1"; + Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); + PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().values().iterator().next(); + List actualValues = ((ListPartitionItem) partitionItem).getItems().get(0) + .getPartitionValuesAsStringList(); + Assert.assertEquals(Arrays.asList( + "dataset/team-a/segment-01", + "/ymd=20260701/hour=[0-9][0-9]/*.jsonl", + "s1"), actualValues); + } + + @Test + public void testGeneratePartitionInfoUsesPartitionColumnOrder() { + List partitionColumns = Arrays.asList( + new Column("source", Type.STRING), + new Column("part_str", Type.STRING), + new Column("pass", Type.STRING)); + Map spec = new LinkedHashMap<>(); + spec.put("pass", "s1"); + spec.put("part_str", "/ymd=20260721"); + spec.put("source", "dataset/team-a/segment-01"); + Partition partition = new Partition(spec, 1L, 1L, 1L, 1L, false); + + PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( + partitionColumns, Collections.singletonList(partition), false); + + String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" + + "/part_str=%2Fymd%3D20260721/pass=s1"; + Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); + PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().get(partitionName); + List actualValues = ((ListPartitionItem) partitionItem).getItems().get(0) + .getPartitionValuesAsStringList(); + Assert.assertEquals(Arrays.asList( + "dataset/team-a/segment-01", "/ymd=20260721", "s1"), actualValues); + } + + @Test + public void testGeneratePartitionInfoPreservesLegacyDateConversion() { + List partitionColumns = Collections.singletonList(new Column("dt", Type.DATEV2)); + Map spec = new LinkedHashMap<>(); + spec.put("dt", "19737"); + Partition partition = new Partition(spec, 1L, 1L, 1L, 1L, false); + + PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( + partitionColumns, Collections.singletonList(partition), true); + + String partitionName = "dt=2024-01-15"; + Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); + PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().get(partitionName); + Assert.assertEquals(Collections.singletonList("2024-01-15"), + ((ListPartitionItem) partitionItem).getItems().get(0).getPartitionValuesAsStringList()); + } + + @Test + public void testGeneratePartitionInfoUsesCollisionFreePartitionNames() { + List partitionColumns = Arrays.asList( + new Column("a", Type.STRING), + new Column("b", Type.STRING)); + Map firstSpec = new LinkedHashMap<>(); + firstSpec.put("a", "x/b=y"); + firstSpec.put("b", "z"); + Map secondSpec = new LinkedHashMap<>(); + secondSpec.put("a", "x"); + secondSpec.put("b", "y/b=z"); + Partition firstPartition = new Partition(firstSpec, 1L, 1L, 1L, 1L, false); + Partition secondPartition = new Partition(secondSpec, 2L, 2L, 2L, 2L, false); + + PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( + partitionColumns, Arrays.asList(firstPartition, secondPartition), false); + + String firstPartitionName = "a=x%2Fb%3Dy/b=z"; + String secondPartitionName = "a=x/b=y%2Fb%3Dz"; + Assert.assertFalse(partitionInfo.isPartitionInvalid()); + Assert.assertEquals(2, partitionInfo.getNameToPartition().size()); + Assert.assertEquals(2, partitionInfo.getNameToPartitionItem().size()); + Assert.assertSame(firstPartition, partitionInfo.getNameToPartition().get(firstPartitionName)); + Assert.assertSame(secondPartition, partitionInfo.getNameToPartition().get(secondPartitionName)); + Assert.assertEquals(Arrays.asList("x/b=y", "z"), + ((ListPartitionItem) partitionInfo.getNameToPartitionItem().get(firstPartitionName)) + .getItems().get(0).getPartitionValuesAsStringList()); + Assert.assertEquals(Arrays.asList("x", "y/b=z"), + ((ListPartitionItem) partitionInfo.getNameToPartitionItem().get(secondPartitionName)) + .getItems().get(0).getPartitionValuesAsStringList()); + } + + @Test + public void testGeneratePartitionInfoRejectsDuplicatePartitionNames() { + List partitionColumns = Collections.singletonList(new Column("part", Type.STRING)); + Map firstSpec = Collections.singletonMap("part", "same"); + Map secondSpec = Collections.singletonMap("part", "same"); + Partition firstPartition = new Partition(firstSpec, 1L, 1L, 1L, 1L, false); + Partition secondPartition = new Partition(secondSpec, 2L, 2L, 2L, 2L, false); + + IllegalStateException exception = Assert.assertThrows(IllegalStateException.class, + () -> PaimonUtil.generatePartitionInfo( + partitionColumns, Arrays.asList(firstPartition, secondPartition), false)); + + Assert.assertTrue(exception.getMessage().contains("Duplicate Paimon partition name")); + } + @Test public void testBinlogHistorySchemaWithSequenceNumber() { PaimonSysExternalTable binlogTable = Mockito.mock(PaimonSysExternalTable.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 034be2185e20f9..9f8583ed1bc860 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.paimon.source; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.common.ExceptionChecker; @@ -35,6 +36,7 @@ import org.apache.doris.qe.SessionVariable; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileScanRangeParams; +import org.apache.doris.thrift.TPushAggOp; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.io.DataFileMeta; @@ -68,6 +70,38 @@ public class PaimonScanNodeTest { @Mock private PaimonFileExternalCatalog paimonFileExternalCatalog; + @Test + public void testCountColumnKeepsAllSplitsWhileCountStarUsesMergedRowCount() throws UserException { + PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new TupleId(3), sv)); + node.setSource(mockPaimonSourceWithPartitionKeys(Collections.emptyList())); + List dataSplits = Arrays.asList( + mockCountDataSplit("f1.parquet", 4_000), + mockCountDataSplit("f2.parquet", 5_000), + mockCountDataSplit("f3.parquet", 6_000)); + Mockito.doReturn(dataSplits).when(node).getPaimonSplitFromAPI(); + Mockito.when(sv.isForceJniScanner()).thenReturn(true); + Mockito.when(sv.getIgnoreSplitType()).thenReturn("NONE"); + Mockito.when(sv.getParallelExecInstanceNum(ArgumentMatchers.nullable(String.class))).thenReturn(1); + + // Before the fix, the raw COUNT opcode made this path keep only parallel representative + // splits and attach the 15,000 metadata rows. BE rejects that shortcut for COUNT(col), so + // it would scan only those representatives and silently miss the discarded DataSplits. + node.setPushDownAggNoGrouping(TPushAggOp.COUNT); + node.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7))); + List countColumnSplits = node.getSplits(1); + Assert.assertEquals(3, countColumnSplits.size()); + for (org.apache.doris.spi.Split split : countColumnSplits) { + Assert.assertFalse(((PaimonSplit) split).getRowCount().isPresent()); + } + + // COUNT(*) remains metadata-only. The 15,000 rows exceed the parallel threshold, so one + // configured execution instance retains one representative split carrying the full sum. + node.setPushDownCountSlotIds(Collections.emptyList()); + List countStarSplits = node.getSplits(1); + Assert.assertEquals(1, countStarSplits.size()); + Assert.assertEquals(Optional.of(15_000L), ((PaimonSplit) countStarSplits.get(0)).getRowCount()); + } + @Test public void testSplitWeight() throws UserException { @@ -692,4 +726,20 @@ private DataSplit createDataSplit(String fileName) { .withDataFiles(Collections.singletonList(dataFileMeta)) .build(); } + + private DataSplit mockCountDataSplit(String fileName, long rowCount) { + DataFileMeta dataFileMeta = DataFileMeta.forAppend(fileName, 64L * 1024 * 1024, rowCount, + SimpleStats.EMPTY_STATS, 1L, 1L, 1L, Collections.emptyList(), null, + FileSource.APPEND, Collections.emptyList(), null, null, + Collections.emptyList()); + DataSplit dataSplit = Mockito.mock(DataSplit.class); + Mockito.when(dataSplit.rowCount()).thenReturn(rowCount); + Mockito.when(dataSplit.mergedRowCountAvailable()).thenReturn(true); + Mockito.when(dataSplit.mergedRowCount()).thenReturn(rowCount); + Mockito.when(dataSplit.partition()).thenReturn(BinaryRow.singleColumn(1)); + Mockito.when(dataSplit.dataFiles()).thenReturn(Collections.singletonList(dataFileMeta)); + Mockito.when(dataSplit.convertToRawFiles()).thenReturn(Optional.empty()); + Mockito.when(dataSplit.deletionFiles()).thenReturn(Optional.empty()); + return dataSplit; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java index 8cf98daea94c59..8ea9041480dadd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java @@ -17,14 +17,20 @@ package org.apache.doris.datasource.tvf.source; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.FunctionGenTable; +import org.apache.doris.datasource.FileQueryScanNode; +import org.apache.doris.datasource.FileSplitter; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.spi.Split; import org.apache.doris.tablefunction.ExternalFileTableValuedFunction; import org.apache.doris.thrift.TBrokerFileStatus; +import org.apache.doris.thrift.TFileType; +import org.apache.doris.thrift.TPushAggOp; import org.junit.Assert; import org.junit.Test; @@ -37,6 +43,40 @@ public class TVFScanNodeTest { private static final long MB = 1024L * 1024L; + @Test + public void testCountColumnKeepsNormalFileSplitting() throws Exception { + SessionVariable sv = new SessionVariable(); + sv.parallelExecInstanceNum = 1; + sv.setFileSplitSize(32 * MB); + TupleDescriptor desc = new TupleDescriptor(new TupleId(0)); + FunctionGenTable table = Mockito.mock(FunctionGenTable.class); + ExternalFileTableValuedFunction tvf = Mockito.mock(ExternalFileTableValuedFunction.class); + Mockito.when(table.getTvf()).thenReturn(tvf); + Mockito.when(tvf.getTFileType()).thenReturn(TFileType.FILE_LOCAL); + Mockito.when(tvf.getFileStatuses()).thenReturn(List.of( + splittableFile("file:///tmp/count_col_1.parquet", 128 * MB), + splittableFile("file:///tmp/count_col_2.parquet", 128 * MB))); + desc.setTable(table); + + TVFScanNode countColumnNode = new TVFScanNode( + new PlanNodeId(0), desc, false, sv, ScanContext.EMPTY); + setFileSplitter(countColumnNode, new FileSplitter(32 * MB, 32 * MB, 0)); + countColumnNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countColumnNode.setPushDownCountSlotIds(Collections.singletonList(new SlotId(7))); + + List countColumnSplits = countColumnNode.getSplits(1); + Assert.assertEquals(8, countColumnSplits.size()); + + TVFScanNode countStarNode = new TVFScanNode( + new PlanNodeId(1), desc, false, sv, ScanContext.EMPTY); + setFileSplitter(countStarNode, new FileSplitter(32 * MB, 32 * MB, 0)); + countStarNode.setPushDownAggNoGrouping(TPushAggOp.COUNT); + countStarNode.setPushDownCountSlotIds(Collections.emptyList()); + + List countStarSplits = countStarNode.getSplits(1); + Assert.assertEquals(2, countStarSplits.size()); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); @@ -57,4 +97,19 @@ public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Excep long target = (long) method.invoke(node, statuses); Assert.assertEquals(100 * MB, target); } + + private static TBrokerFileStatus splittableFile(String path, long size) { + TBrokerFileStatus status = new TBrokerFileStatus(); + status.setPath(path); + status.setSize(size); + status.setModificationTime(0); + status.setIsSplitable(true); + return status; + } + + private static void setFileSplitter(TVFScanNode node, FileSplitter splitter) throws Exception { + java.lang.reflect.Field field = FileQueryScanNode.class.getDeclaredField("fileSplitter"); + field.setAccessible(true); + field.set(node, splitter); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/SpiSwitchingFileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/SpiSwitchingFileSystemTest.java index 2abe4541100815..3294a0bb0095ce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/SpiSwitchingFileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/SpiSwitchingFileSystemTest.java @@ -20,7 +20,12 @@ import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.property.storage.BrokerProperties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.filesystem.DorisInputFile; +import org.apache.doris.filesystem.DorisInputStream; +import org.apache.doris.filesystem.FileEntry; +import org.apache.doris.filesystem.FileIterator; import org.apache.doris.filesystem.FileSystem; +import org.apache.doris.filesystem.Location; import com.google.common.collect.Maps; import org.junit.Assert; @@ -32,7 +37,10 @@ import java.lang.reflect.Field; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -221,6 +229,130 @@ public void close() throws IOException { } } + // ----------------------------------------------------------------------- + // Legacy cross-scheme fallback translation + // ----------------------------------------------------------------------- + + /** + * When the legacy fallback routes a {@code cos://} path to an S3-typed storage + * (LocationPath.findStorageProperties step 3), the delegate filesystem must receive + * the storage's native {@code s3://} scheme (its URI parser whitelists only its own + * schemes), while every path returned to the caller must keep the original + * {@code cos://} scheme so comparisons against HMS/Iceberg metadata still hold. + */ + @Test + public void testCompatFallbackTranslatesUrisBothWays() throws Exception { + StorageProperties s3Props = Mockito.mock(StorageProperties.class); + Mockito.when(s3Props.getType()).thenReturn(StorageProperties.Type.S3); + CapturingFileSystem fake = new CapturingFileSystem(); + + try (MockedStatic mockedLocationPath = + Mockito.mockStatic(LocationPath.class, Mockito.CALLS_REAL_METHODS); + MockedStatic mockedFactory = + Mockito.mockStatic(FileSystemFactory.class)) { + // Simulate the fallback: cos:// path served by an S3-typed storage whose + // validateAndNormalizeUri rewrote the scheme prefix to s3://. + mockedLocationPath.when(() -> LocationPath.of(Mockito.anyString(), Mockito.anyMap())) + .thenAnswer(inv -> { + String location = inv.getArgument(0); + String normalized = "s3://" + location.substring("cos://".length()); + return LocationPath.ofDirect(normalized, "cos", "s3://bucket", s3Props); + }); + mockedFactory.when(() -> FileSystemFactory.getFileSystem( + Mockito.any(StorageProperties.class))) + .thenReturn(fake); + + SpiSwitchingFileSystem spiFs = new SpiSwitchingFileSystem(Collections.emptyMap()); + + // Inbound: single-location operations are translated to the native scheme. + spiFs.exists(Location.of("cos://bucket/dir/file1")); + Assert.assertEquals("s3://bucket/dir/file1", fake.lastUri); + + // Inbound: rename translates both endpoints. + spiFs.rename(Location.of("cos://bucket/dir/a"), Location.of("cos://bucket/dir/b")); + Assert.assertEquals("s3://bucket/dir/a", fake.lastRenameSrc); + Assert.assertEquals("s3://bucket/dir/b", fake.lastRenameDst); + + // Outbound: listing results are translated back to the caller's scheme. + fake.entries = List.of( + new FileEntry(Location.of("s3://bucket/dir/f1"), 1, false, 0, null), + new FileEntry(Location.of("s3://bucket/dir/sub/"), 0, true, 0, null)); + List files = spiFs.listFiles(Location.of("cos://bucket/dir")); + Assert.assertEquals("s3://bucket/dir", fake.lastUri); + Assert.assertEquals(1, files.size()); + Assert.assertEquals("cos://bucket/dir/f1", files.get(0).location().uri()); + + Set dirs = spiFs.listDirectories(Location.of("cos://bucket/dir")); + Assert.assertEquals(Collections.singleton("cos://bucket/dir/sub/"), dirs); + + // Outbound: newInputFile reports the caller's location, while the delegate + // was opened with the native scheme. + DorisInputFile inputFile = spiFs.newInputFile(Location.of("cos://bucket/dir/f1")); + Assert.assertEquals("s3://bucket/dir/f1", fake.lastUri); + Assert.assertEquals("cos://bucket/dir/f1", inputFile.location().uri()); + } + } + + /** + * A direct type match (cos:// path on a COS-typed storage) must NOT be translated, + * even though normalization rewrites the scheme — the delegate natively accepts the + * caller's scheme and must keep seeing it (today's behavior, byte for byte). + */ + @Test + public void testDirectMatchSkipsTranslation() throws Exception { + StorageProperties cosProps = Mockito.mock(StorageProperties.class); + Mockito.when(cosProps.getType()).thenReturn(StorageProperties.Type.COS); + CapturingFileSystem fake = new CapturingFileSystem(); + + try (MockedStatic mockedLocationPath = + Mockito.mockStatic(LocationPath.class, Mockito.CALLS_REAL_METHODS); + MockedStatic mockedFactory = + Mockito.mockStatic(FileSystemFactory.class)) { + mockedLocationPath.when(() -> LocationPath.of(Mockito.anyString(), Mockito.anyMap())) + .thenAnswer(inv -> { + String location = inv.getArgument(0); + String normalized = "s3://" + location.substring("cos://".length()); + return LocationPath.ofDirect(normalized, "cos", "s3://bucket", cosProps); + }); + mockedFactory.when(() -> FileSystemFactory.getFileSystem( + Mockito.any(StorageProperties.class))) + .thenReturn(fake); + + SpiSwitchingFileSystem spiFs = new SpiSwitchingFileSystem(Collections.emptyMap()); + spiFs.exists(Location.of("cos://bucket/dir/file1")); + Assert.assertEquals("cos://bucket/dir/file1", fake.lastUri); + } + } + + /** + * A cross-type fallback whose normalization does not change the scheme (e.g. an + * s3:// path served by a MinIO-typed storage) must not be translated either. + */ + @Test + public void testSameSchemeFallbackSkipsTranslation() throws Exception { + StorageProperties minioProps = Mockito.mock(StorageProperties.class); + Mockito.when(minioProps.getType()).thenReturn(StorageProperties.Type.MINIO); + CapturingFileSystem fake = new CapturingFileSystem(); + + try (MockedStatic mockedLocationPath = + Mockito.mockStatic(LocationPath.class, Mockito.CALLS_REAL_METHODS); + MockedStatic mockedFactory = + Mockito.mockStatic(FileSystemFactory.class)) { + mockedLocationPath.when(() -> LocationPath.of(Mockito.anyString(), Mockito.anyMap())) + .thenAnswer(inv -> { + String location = inv.getArgument(0); + return LocationPath.ofDirect(location, "s3", "s3://bucket", minioProps); + }); + mockedFactory.when(() -> FileSystemFactory.getFileSystem( + Mockito.any(StorageProperties.class))) + .thenReturn(fake); + + SpiSwitchingFileSystem spiFs = new SpiSwitchingFileSystem(Collections.emptyMap()); + spiFs.exists(Location.of("s3://bucket/key")); + Assert.assertEquals("s3://bucket/key", fake.lastUri); + } + } + // ----------------------------------------------------------------------- // Helpers // ----------------------------------------------------------------------- @@ -239,6 +371,72 @@ private static void injectIntoCache(SpiSwitchingFileSystem spiFs, cache.put(key, value); } + /** + * Records the URIs the delegate actually receives and serves canned listing entries, + * so tests can assert on both directions of the scheme translation. + */ + private static class CapturingFileSystem extends StubFileSystem { + String lastUri; + String lastRenameSrc; + String lastRenameDst; + List entries = List.of(); + + @Override + public boolean exists(Location location) throws IOException { + lastUri = location.uri(); + return true; + } + + @Override + public void rename(Location src, Location dst) throws IOException { + lastRenameSrc = src.uri(); + lastRenameDst = dst.uri(); + } + + @Override + public FileIterator list(Location location) throws IOException { + lastUri = location.uri(); + Iterator it = entries.iterator(); + return new FileIterator() { + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public FileEntry next() { + return it.next(); + } + + @Override + public void close() { + // nothing to release + } + }; + } + + @Override + public DorisInputFile newInputFile(Location location) throws IOException { + lastUri = location.uri(); + return new DorisInputFile() { + @Override + public Location location() { + return location; + } + + @Override + public long length() { + return 0; + } + + @Override + public DorisInputStream newStream() { + throw new UnsupportedOperationException(); + } + }; + } + } + /** * Minimal no-op stub that satisfies the {@link FileSystem} interface. * All methods throw {@link UnsupportedOperationException} except close(), which is a no-op. diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/StoragePropertiesConverterHdfsDirTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/StoragePropertiesConverterHdfsDirTest.java new file mode 100644 index 00000000000000..30dd85a87c27f4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/StoragePropertiesConverterHdfsDirTest.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fs; + +import org.apache.doris.common.Config; +import org.apache.doris.datasource.property.storage.HdfsProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +public class StoragePropertiesConverterHdfsDirTest { + + @Test + public void hdfsMapCarriesInjectedConfigDir() { + Map raw = new HashMap<>(); + raw.put("fs.defaultFS", "hdfs://ns"); + HdfsProperties props = new HdfsProperties(raw); + props.initNormalizeAndCheckProps(); + + Map map = StoragePropertiesConverter.toMap(props); + + Assertions.assertEquals("HDFS", map.get("_STORAGE_TYPE_")); + Assertions.assertEquals(Config.hadoop_config_dir, map.get("_HADOOP_CONFIG_DIR_")); + } + + @Test + public void ossHdfsMapCarriesDistinctStorageType() { + // OSS-HDFS extends the HDFS-compatible base but must carry the OSS_HDFS marker so the + // fe-filesystem OssHdfsFileSystemProvider (not the plain HDFS one) is selected. + Map raw = new HashMap<>(); + raw.put("oss.hdfs.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + raw.put("oss.hdfs.access_key", "ak"); + raw.put("oss.hdfs.secret_key", "sk"); + StorageProperties props = StorageProperties.createPrimary(raw); + + Map map = StoragePropertiesConverter.toMap(props); + + Assertions.assertEquals("OSS_HDFS", map.get("_STORAGE_TYPE_")); + Assertions.assertEquals(Config.hadoop_config_dir, map.get("_HADOOP_CONFIG_DIR_")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/httpv2/meta/MetaServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/httpv2/meta/MetaServiceTest.java new file mode 100644 index 00000000000000..90bb54de6b9e43 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/httpv2/meta/MetaServiceTest.java @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.meta; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.ha.FrontendNodeType; +import org.apache.doris.httpv2.controller.BaseController; +import org.apache.doris.httpv2.entity.ResponseBody; +import org.apache.doris.httpv2.exception.UnauthorizedException; +import org.apache.doris.httpv2.rest.RestApiStatusCode; +import org.apache.doris.master.MetaHelper; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.Storage; +import org.apache.doris.system.Frontend; + +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.springframework.http.ResponseEntity; + +import java.io.File; +import java.lang.reflect.Field; + +public class MetaServiceTest { + // Value configured via Config.fe_meta_auth_token -- the cluster meta auth token. + private static final String META_TOKEN = "meta-auth-token"; + // Token persisted in the local Storage, returned by the /check endpoint. Intentionally + // different from META_TOKEN to show the two are independent. + private static final String STORAGE_TOKEN = "storage-token"; + private static final String BAD_TOKEN = "bad-token"; + private static final String FE_HOST = "127.0.0.1"; + private static final int FE_EDIT_LOG_PORT = 9010; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private String oldMetaAuthToken; + private boolean oldEnableAllHttpAuth; + private int oldHttpPort; + private int oldHttpsPort; + private boolean oldEnableHttps; + private Env env; + private MockedStatic envStatic; + + @Before + public void setUp() { + oldMetaAuthToken = Config.fe_meta_auth_token; + oldEnableAllHttpAuth = Config.enable_all_http_auth; + oldHttpPort = Config.http_port; + oldHttpsPort = Config.https_port; + oldEnableHttps = Config.enable_https; + // Token required by default; individual tests clear it to exercise the legacy path. + Config.fe_meta_auth_token = META_TOKEN; + Config.enable_all_http_auth = false; + Config.http_port = 8030; + Config.https_port = 8050; + Config.enable_https = false; + + env = Mockito.mock(Env.class); + envStatic = Mockito.mockStatic(Env.class); + envStatic.when(Env::getCurrentEnv).thenReturn(env); + + Frontend frontend = new Frontend(FrontendNodeType.FOLLOWER, "fe1", FE_HOST, FE_EDIT_LOG_PORT); + Mockito.when(env.checkFeExist(FE_HOST, FE_EDIT_LOG_PORT)).thenReturn(frontend); + Mockito.when(env.getImageDir()).thenReturn(temporaryFolder.getRoot().getAbsolutePath()); + } + + @After + public void tearDown() { + Config.fe_meta_auth_token = oldMetaAuthToken; + Config.enable_all_http_auth = oldEnableAllHttpAuth; + Config.http_port = oldHttpPort; + Config.https_port = oldHttpsPort; + Config.enable_https = oldEnableHttps; + if (envStatic != null) { + envStatic.close(); + } + } + + // A matching token passes even when the request originates from a proxy address + // (remoteAddr differs from the registered FE host). + @Test + public void testMatchingTokenAllowsMetaRequestFromProxyAddress() throws Exception { + MetaService service = new MetaService(); + HttpServletRequest request = newRequest("192.0.2.10", FE_HOST, META_TOKEN); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + Object result = service.role(request, response); + + Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result)); + Mockito.verify(response).setHeader("role", FrontendNodeType.FOLLOWER.name()); + } + + // When no token is configured, the node-host check alone is sufficient. This keeps + // existing clusters and rolling upgrades working (a peer that sends no token is allowed). + @Test + public void testNoTokenRequiredWhenTokenNotConfigured() throws Exception { + Config.fe_meta_auth_token = ""; + MetaService service = new MetaService(); + HttpServletRequest request = newRequest("192.0.2.10", FE_HOST, null); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + Object result = service.role(request, response); + + Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result)); + Mockito.verify(response).setHeader("role", FrontendNodeType.FOLLOWER.name()); + } + + // /check returns the local Storage token once the request passes authentication. + @Test + public void testCheckReturnsStorageTokenWhenAuthPasses() throws Exception { + Config.fe_meta_auth_token = ""; + MetaService service = serviceWithImageDir(); + HttpServletRequest request = newRequest("192.0.2.10", FE_HOST, null); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + Object result = service.check(request, response); + + Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result)); + Mockito.verify(response).setHeader(MetaBaseAction.TOKEN, STORAGE_TOKEN); + } + + // A wrong token is rejected even though the node-host check would pass. + @Test + public void testWrongTokenRejected() { + MetaService service = new MetaService(); + HttpServletRequest request = newRequest(FE_HOST, FE_HOST, BAD_TOKEN); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + Assert.assertThrows(UnauthorizedException.class, () -> service.role(request, response)); + } + + // A missing token is rejected when a cluster token is configured. + @Test + public void testMissingTokenRejectedWhenConfigured() { + MetaService service = new MetaService(); + HttpServletRequest request = newRequest(FE_HOST, FE_HOST, null); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + Assert.assertThrows(UnauthorizedException.class, () -> service.role(request, response)); + } + + // The node-host check is always enforced: an unknown FE host is rejected even with a + // matching token (the token check is additive, it does not replace the host check). + @Test + public void testUnknownHostRejectedEvenWithValidToken() { + MetaService service = new MetaService(); + HttpServletRequest request = newRequest("192.0.2.99", "192.0.2.99", META_TOKEN); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + Assert.assertThrows(UnauthorizedException.class, () -> service.role(request, response)); + } + + @Test + public void testPutRejectsUnexpectedHttpPort() throws Exception { + MetaService service = new MetaService(); + HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Mockito.when(request.getParameter("version")).thenReturn("100"); + Mockito.when(request.getParameter("port")).thenReturn(Integer.toString(Config.http_port + 1)); + + try (MockedStatic metaHelper = Mockito.mockStatic(MetaHelper.class)) { + File partialFile = temporaryFolder.newFile("image.100.part"); + metaHelper.when(() -> MetaHelper.getFile(Mockito.anyString(), Mockito.any(File.class))) + .thenReturn(partialFile); + + Object result = service.put(request, response); + + Assert.assertEquals(RestApiStatusCode.BAD_REQUEST.code, responseCode(result)); + metaHelper.verify(() -> MetaHelper.getRemoteFile(Mockito.anyString(), Mockito.anyInt(), + Mockito.any(File.class)), Mockito.never()); + } + } + + // When HTTPS is enabled the master pushes image using https_port, so /put must accept + // https_port. The expected port follows HttpURLUtil.getHttpPort(), not a hard-coded http_port. + @Test + public void testPutAcceptsHttpsPortWhenHttpsEnabled() throws Exception { + Config.enable_https = true; + MetaService service = new MetaService(); + HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Mockito.when(request.getParameter("version")).thenReturn("100"); + Mockito.when(request.getParameter("port")).thenReturn(Integer.toString(Config.https_port)); + + try (MockedStatic metaHelper = Mockito.mockStatic(MetaHelper.class)) { + File partialFile = temporaryFolder.newFile("image.100.part"); + metaHelper.when(() -> MetaHelper.getFile(Mockito.anyString(), Mockito.any(File.class))) + .thenReturn(partialFile); + + Object result = service.put(request, response); + + // Passes the port check and proceeds to fetch the remote image (no BAD_REQUEST). + Assert.assertEquals(RestApiStatusCode.OK.code, responseCode(result)); + metaHelper.verify(() -> MetaHelper.getRemoteFile(Mockito.anyString(), Mockito.anyInt(), + Mockito.any(File.class)), Mockito.times(1)); + } + } + + // /dump authenticates and then requires ADMIN. An authenticated admin passes the privilege + // check and proceeds to dump. + @Test + public void testDumpAllowsAdmin() throws Exception { + MetaService service = Mockito.spy(new MetaService()); + HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + Mockito.doReturn(authInfo(UserIdentity.ADMIN)).when(service).executeCheckPassword(request, response); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.when(env.getAccessManager()).thenReturn(accessManager); + Mockito.when(accessManager.checkGlobalPriv(UserIdentity.ADMIN, PrivPredicate.ADMIN)).thenReturn(true); + Mockito.when(env.dumpImage()).thenReturn(null); + + service.dump(request, response); + + Mockito.verify(service).executeCheckPassword(request, response); + Mockito.verify(env).dumpImage(); + } + + // An authenticated but non-admin user is rejected before any dump happens. + @Test + public void testDumpRejectsNonAdmin() throws Exception { + MetaService service = Mockito.spy(new MetaService()); + HttpServletRequest request = newRequest(FE_HOST, FE_HOST, META_TOKEN); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + UserIdentity nonAdmin = UserIdentity.createAnalyzedUserIdentWithIp("user", "%"); + Mockito.doReturn(authInfo(nonAdmin)).when(service).executeCheckPassword(request, response); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.when(env.getAccessManager()).thenReturn(accessManager); + Mockito.when(accessManager.checkGlobalPriv(nonAdmin, PrivPredicate.ADMIN)).thenReturn(false); + + Assert.assertThrows(UnauthorizedException.class, () -> service.dump(request, response)); + Mockito.verify(env, Mockito.never()).dumpImage(); + } + + private static BaseController.ActionAuthorizationInfo authInfo(UserIdentity userIdentity) { + BaseController.ActionAuthorizationInfo info = new BaseController.ActionAuthorizationInfo(); + info.userIdentity = userIdentity; + return info; + } + + private MetaService serviceWithImageDir() throws Exception { + File imageDir = temporaryFolder.newFolder("image"); + Storage storage = new Storage(12345, STORAGE_TOKEN, imageDir.getAbsolutePath()); + storage.writeClusterIdAndToken(); + + MetaService service = new MetaService(); + Field imageDirField = MetaService.class.getDeclaredField("imageDir"); + imageDirField.setAccessible(true); + imageDirField.set(service, imageDir); + return service; + } + + private HttpServletRequest newRequest(String remoteAddr, String clientHost, String token) { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + Mockito.when(request.getHeader(Env.CLIENT_NODE_HOST_KEY)).thenReturn(clientHost); + Mockito.when(request.getHeader(Env.CLIENT_NODE_PORT_KEY)).thenReturn(Integer.toString(FE_EDIT_LOG_PORT)); + Mockito.when(request.getHeader(MetaBaseAction.TOKEN)).thenReturn(token); + Mockito.when(request.getRemoteAddr()).thenReturn(remoteAddr); + Mockito.when(request.getRemoteHost()).thenReturn(remoteAddr); + Mockito.when(request.getParameter("host")).thenReturn(clientHost); + Mockito.when(request.getParameter("port")).thenReturn(Integer.toString(FE_EDIT_LOG_PORT)); + return request; + } + + private int responseCode(Object result) { + ResponseEntity responseEntity = (ResponseEntity) result; + ResponseBody body = (ResponseBody) responseEntity.getBody(); + return body.getCode(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/LoadActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/LoadActionTest.java index 2c8c59d413a786..5226aa66a64da6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/LoadActionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/LoadActionTest.java @@ -21,6 +21,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TNetworkAddress; @@ -253,6 +254,30 @@ public void testGetAllHeadersMasksSensitiveHeaders() throws Exception { Assertions.assertTrue(headers.contains("label:load_label")); } + @Test + public void testGetCloudClusterNamePrefersComputeGroupHeader() throws Exception { + LoadAction loadAction = new LoadAction(); + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + Mockito.when(request.getHeader(SessionVariable.COMPUTE_GROUP)).thenReturn("compute_group_1"); + Mockito.when(request.getHeader(SessionVariable.CLOUD_CLUSTER)).thenReturn("cloud_cluster_1"); + + String cloudClusterName = invokeGetCloudClusterName(loadAction, request); + + Assertions.assertEquals("compute_group_1", cloudClusterName); + } + + @Test + public void testGetCloudClusterNameFallsBackToCloudClusterHeader() throws Exception { + LoadAction loadAction = new LoadAction(); + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + Mockito.when(request.getHeader(SessionVariable.COMPUTE_GROUP)).thenReturn(""); + Mockito.when(request.getHeader(SessionVariable.CLOUD_CLUSTER)).thenReturn("cloud_cluster_1"); + + String cloudClusterName = invokeGetCloudClusterName(loadAction, request); + + Assertions.assertEquals("cloud_cluster_1", cloudClusterName); + } + @Test public void testExecuteWithoutPasswordRedirectsToBackend() throws Exception { Config.enable_debug_points = true; @@ -453,6 +478,12 @@ private RedirectView invokeRedirectToStreamLoadForward(LoadAction loadAction, Ht return (RedirectView) method.invoke(loadAction, request, addr, forwardTarget); } + private String invokeGetCloudClusterName(LoadAction loadAction, HttpServletRequest request) throws Exception { + Method method = LoadAction.class.getDeclaredMethod("getCloudClusterName", HttpServletRequest.class); + method.setAccessible(true); + return (String) method.invoke(loadAction, request); + } + private TNetworkAddress invokeSelectEndpointByRedirectPolicy(LoadAction loadAction, HttpServletRequest request, Backend backend) throws Exception { Method method = LoadAction.class.getDeclaredMethod("selectEndpointByRedirectPolicy", diff --git a/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/RestBaseControllerTest.java b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/RestBaseControllerTest.java index b838e25ef50a21..a42a389c911959 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/RestBaseControllerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/RestBaseControllerTest.java @@ -54,11 +54,30 @@ public void testBuildRedirectUrlWithoutQueryString() { Assert.assertEquals("http://be-host:8040/api/db%2Ftbl/_stream_load", redirectUrl); } + @Test + public void testBuildRedirectUrlToBackendForcesHttpEvenWhenRequestIsHttps() { + // BE never terminates TLS, so the redirect must stay "http" regardless of request scheme. + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + Mockito.when(request.getScheme()).thenReturn("https"); + Mockito.when(request.getHeader("Authorization")).thenReturn(null); + + TestRestController controller = new TestRestController(); + String redirectUrl = controller.buildRedirectUrlToBackendForTest(request, + new TNetworkAddress("be-host", 8040), "/api/db/tbl/_stream_load", "k=v"); + + Assert.assertEquals("http://be-host:8040/api/db/tbl/_stream_load?k=v", redirectUrl); + } + // Expose the protected helper so the redirect URL can be verified directly. private static class TestRestController extends RestBaseController { private String buildRedirectUrlForTest(HttpServletRequest request, TNetworkAddress addr, String requestPath, String queryString) { return buildRedirectUrl(request, addr, requestPath, queryString); } + + private String buildRedirectUrlToBackendForTest(HttpServletRequest request, TNetworkAddress addr, + String requestPath, String queryString) { + return buildRedirectUrlToBackend(request, addr, requestPath, queryString); + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/manager/HttpUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/manager/HttpUtilsTest.java new file mode 100644 index 00000000000000..2efaf4ca7bad16 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/manager/HttpUtilsTest.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest.manager; + +import org.apache.doris.common.Config; +import org.apache.doris.common.util.InternalHttpsUtils; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.lang.reflect.Field; + +public class HttpUtilsTest { + + // Port 1 refuses connections on loopback, so no real HTTP(S) server is needed. + private static final String REFUSED_PORT_URL_HTTP = "http://127.0.0.1:1/ping"; + private static final String REFUSED_PORT_URL_HTTPS = "https://127.0.0.1:1/ping"; + + private boolean originalEnableHttps; + private String originalKeyStorePath; + + @Before + public void setUp() throws Exception { + originalEnableHttps = Config.enable_https; + originalKeyStorePath = Config.key_store_path; + resetCachedSslContext(); + } + + @After + public void tearDown() throws Exception { + Config.enable_https = originalEnableHttps; + Config.key_store_path = originalKeyStorePath; + resetCachedSslContext(); + } + + private void resetCachedSslContext() throws Exception { + Field field = InternalHttpsUtils.class.getDeclaredField("cachedSslContext"); + field.setAccessible(true); + field.set(null, null); + } + + @Test + public void testExecuteRequestUsesPlainClientForHttpUrlEvenWhenHttpsEnabled() throws Exception { + Config.enable_https = true; + Config.key_store_path = "/non/existent/path/doris_ssl_certificate.keystore"; + resetCachedSslContext(); + + try { + HttpUtils.doGet(REFUSED_PORT_URL_HTTP, null); + Assert.fail("Expected a connection failure against the refused port"); + } catch (RuntimeException e) { + Assert.fail("Should not have attempted to build the HTTPS client for an http:// URL: " + + e.getMessage()); + } catch (IOException expected) { + // Plain client hit the network and failed there, never touching the broken keystore. + } + } + + @Test + public void testExecuteRequestUsesHttpsClientForHttpsUrl() throws Exception { + Config.enable_https = true; + Config.key_store_path = "/non/existent/path/doris_ssl_certificate.keystore"; + resetCachedSslContext(); + + try { + HttpUtils.doGet(REFUSED_PORT_URL_HTTPS, null); + Assert.fail("Expected SSLContext build failure before any connection attempt"); + } catch (RuntimeException e) { + Assert.assertTrue("Failure should come from the missing keystore, not an unrelated error", + e.getMessage() != null && e.getMessage().contains("doris_ssl_certificate.keystore")); + } catch (IOException e) { + Assert.fail("Expected the HTTPS client's keystore failure, not a network-level error: " + + e.getMessage()); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidatorTest.java index 84cc3bdf5e0e30..3765fd1018933e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidatorTest.java @@ -17,12 +17,20 @@ package org.apache.doris.job.extensions.insert.streaming; +import org.apache.doris.datasource.jdbc.client.JdbcClient; import org.apache.doris.job.cdc.DataSourceConfigKeys; import org.apache.doris.job.common.DataSourceType; +import org.apache.doris.job.exception.JobException; +import org.apache.doris.job.util.StreamingJobUtils; import org.junit.Assert; import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; import java.util.HashMap; import java.util.Map; @@ -398,4 +406,184 @@ public void testServerIdOptional() { props.put(DataSourceConfigKeys.SNAPSHOT_PARALLELISM, "4"); DataSourceConfigValidator.validateSource(props, DataSourceType.MYSQL.name()); } + + @Test + public void testOceanBaseAcceptsMysqlJdbcUrl() { + Map props = new HashMap<>(); + props.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://localhost:2883/test_db"); + + DataSourceConfigValidator.validateSource(props, DataSourceType.OCEANBASE.name()); + } + + @Test + public void testOceanBaseAllowsPartialPropertiesWithoutJdbcUrl() { + Map props = new HashMap<>(); + props.put(DataSourceConfigKeys.SNAPSHOT_PARALLELISM, "2"); + + DataSourceConfigValidator.validateSource(props, DataSourceType.OCEANBASE.name()); + } + + @Test + public void testOceanBaseRejectsNonMysqlJdbcUrl() { + Map props = new HashMap<>(); + props.put(DataSourceConfigKeys.JDBC_URL, "jdbc:oceanbase://localhost:2883/test_db"); + + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> DataSourceConfigValidator.validateSource( + props, DataSourceType.OCEANBASE.name())); + + Assert.assertTrue(exception.getMessage().contains("jdbc:mysql://")); + } + + @Test + public void testOceanBaseRejectsPostgresProperties() { + for (String key : new String[] { + DataSourceConfigKeys.SCHEMA, + DataSourceConfigKeys.SLOT_NAME, + DataSourceConfigKeys.PUBLICATION_NAME}) { + Map props = new HashMap<>(); + props.put(key, "value"); + + IllegalArgumentException exception = Assert.assertThrows( + IllegalArgumentException.class, + () -> DataSourceConfigValidator.validateSource( + props, DataSourceType.OCEANBASE.name())); + Assert.assertTrue(exception.getMessage().contains(key)); + } + } + + @Test + public void testOceanBaseDoesNotExposeSchemaChangeEnabled() { + Map props = new HashMap<>(); + props.put(DataSourceConfigKeys.SCHEMA_CHANGE_ENABLED, "false"); + + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> DataSourceConfigValidator.validateSource( + props, DataSourceType.OCEANBASE.name())); + + Assert.assertTrue(exception.getMessage().contains(DataSourceConfigKeys.SCHEMA_CHANGE_ENABLED)); + } + + @Test + public void testOceanBaseSupportsEarliestOffset() { + Assert.assertTrue(DataSourceConfigValidator.isValidOffset( + DataSourceConfigKeys.OFFSET_EARLIEST, DataSourceType.OCEANBASE.name())); + Assert.assertFalse(DataSourceConfigValidator.isValidOffset( + DataSourceConfigKeys.OFFSET_EARLIEST, DataSourceType.POSTGRES.name())); + } + + @Test + public void testCompatibilityModePreflightIsNoopForExistingSources() throws Exception { + DataSourceConfigValidator.validateSourceBeforeTableCreation( + DataSourceType.MYSQL, new HashMap<>()); + DataSourceConfigValidator.validateSourceBeforeTableCreation( + DataSourceType.POSTGRES, new HashMap<>()); + } + + @Test + public void testOceanBaseMysqlCompatibilityModePasses() throws Exception { + JdbcClient jdbcClient = mockOceanBaseCompatibilityMode(true, "MYSQL"); + + try (MockedStatic utils = Mockito.mockStatic(StreamingJobUtils.class)) { + utils.when(() -> StreamingJobUtils.getJdbcClient( + Mockito.eq(DataSourceType.OCEANBASE), Mockito.anyMap())) + .thenReturn(jdbcClient); + + DataSourceConfigValidator.validateSourceBeforeTableCreation( + DataSourceType.OCEANBASE, new HashMap<>()); + } + + Mockito.verify(jdbcClient).closeClient(); + } + + @Test + public void testOceanBaseOracleCompatibilityModeIsRejected() throws Exception { + JdbcClient jdbcClient = mockOceanBaseCompatibilityMode(true, "ORACLE"); + + try (MockedStatic utils = Mockito.mockStatic(StreamingJobUtils.class)) { + utils.when(() -> StreamingJobUtils.getJdbcClient( + Mockito.eq(DataSourceType.OCEANBASE), Mockito.anyMap())) + .thenReturn(jdbcClient); + + JobException exception = Assert.assertThrows(JobException.class, + () -> DataSourceConfigValidator.validateSourceBeforeTableCreation( + DataSourceType.OCEANBASE, new HashMap<>())); + Assert.assertTrue(exception.getMessage().contains("Oracle compatibility mode")); + } + + Mockito.verify(jdbcClient).closeClient(); + } + + @Test + public void testOceanBaseUnknownCompatibilityModeIsRejected() throws Exception { + JdbcClient jdbcClient = mockOceanBaseCompatibilityMode(true, "UNKNOWN"); + + try (MockedStatic utils = Mockito.mockStatic(StreamingJobUtils.class)) { + utils.when(() -> StreamingJobUtils.getJdbcClient( + Mockito.eq(DataSourceType.OCEANBASE), Mockito.anyMap())) + .thenReturn(jdbcClient); + + JobException exception = Assert.assertThrows(JobException.class, + () -> DataSourceConfigValidator.validateSourceBeforeTableCreation( + DataSourceType.OCEANBASE, new HashMap<>())); + Assert.assertTrue(exception.getMessage().contains("UNKNOWN")); + } + + Mockito.verify(jdbcClient).closeClient(); + } + + @Test + public void testOceanBaseEmptyCompatibilityModeResultIsRejected() throws Exception { + JdbcClient jdbcClient = mockOceanBaseCompatibilityMode(false, null); + + try (MockedStatic utils = Mockito.mockStatic(StreamingJobUtils.class)) { + utils.when(() -> StreamingJobUtils.getJdbcClient( + Mockito.eq(DataSourceType.OCEANBASE), Mockito.anyMap())) + .thenReturn(jdbcClient); + + JobException exception = Assert.assertThrows(JobException.class, + () -> DataSourceConfigValidator.validateSourceBeforeTableCreation( + DataSourceType.OCEANBASE, new HashMap<>())); + Assert.assertTrue(exception.getMessage().contains("Failed to determine")); + } + + Mockito.verify(jdbcClient).closeClient(); + } + + @Test + public void testOceanBaseCompatibilityModeQueryFailurePreservesCause() throws Exception { + JdbcClient jdbcClient = Mockito.mock(JdbcClient.class); + Connection connection = Mockito.mock(Connection.class); + Mockito.when(jdbcClient.getConnection()).thenReturn(connection); + Mockito.when(connection.createStatement()).thenThrow(new IllegalStateException("query failed")); + + try (MockedStatic utils = Mockito.mockStatic(StreamingJobUtils.class)) { + utils.when(() -> StreamingJobUtils.getJdbcClient( + Mockito.eq(DataSourceType.OCEANBASE), Mockito.anyMap())) + .thenReturn(jdbcClient); + + JobException exception = Assert.assertThrows(JobException.class, + () -> DataSourceConfigValidator.validateSourceBeforeTableCreation( + DataSourceType.OCEANBASE, new HashMap<>())); + Assert.assertTrue(exception.getMessage().contains("query failed")); + Assert.assertNotNull(exception.getCause()); + } + + Mockito.verify(jdbcClient).closeClient(); + } + + private JdbcClient mockOceanBaseCompatibilityMode(boolean hasResult, String mode) + throws Exception { + JdbcClient jdbcClient = Mockito.mock(JdbcClient.class); + Connection connection = Mockito.mock(Connection.class); + Statement statement = Mockito.mock(Statement.class); + ResultSet resultSet = Mockito.mock(ResultSet.class); + Mockito.when(jdbcClient.getConnection()).thenReturn(connection); + Mockito.when(connection.createStatement()).thenReturn(statement); + Mockito.when(statement.executeQuery("SHOW VARIABLES LIKE 'ob_compatibility_mode'")) + .thenReturn(resultSet); + Mockito.when(resultSet.next()).thenReturn(hasResult); + Mockito.when(resultSet.getString(2)).thenReturn(mode); + return jdbcClient; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingJdbcUrlNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingJdbcUrlNormalizerTest.java new file mode 100644 index 00000000000000..2dda159a11dcde --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingJdbcUrlNormalizerTest.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.extensions.insert.streaming; + +import org.apache.doris.job.common.DataSourceType; + +import org.junit.Assert; +import org.junit.Test; + +public class StreamingJdbcUrlNormalizerTest { + + @Test + public void testNormalizeMysqlJdbcUrl() { + String jdbcUrl = StreamingJdbcUrlNormalizer.normalize( + DataSourceType.MYSQL, "jdbc:mysql://127.0.0.1:3306/test"); + + Assert.assertEquals("jdbc:mysql://127.0.0.1:3306/test?yearIsDateType=false" + + "&tinyInt1isBit=false&useUnicode=true&characterEncoding=utf-8", + jdbcUrl); + Assert.assertFalse(jdbcUrl.contains("rewriteBatchedStatements")); + } + + @Test + public void testNormalizeMysqlJdbcUrlPreservesExplicitValues() { + String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/test?tinyInt1isBit=true" + + "&yearIsDateType=true&useUnicode=false&characterEncoding=GBK"; + + Assert.assertEquals(jdbcUrl, + StreamingJdbcUrlNormalizer.normalize(DataSourceType.MYSQL, jdbcUrl)); + + String partialJdbcUrl = "jdbc:mysql://127.0.0.1:3306/test?yearIsDateType=true"; + Assert.assertEquals(partialJdbcUrl + "&tinyInt1isBit=false" + + "&useUnicode=true&characterEncoding=utf-8", + StreamingJdbcUrlNormalizer.normalize(DataSourceType.MYSQL, partialJdbcUrl)); + } + + @Test + public void testNormalizeMysqlJdbcUrlMatchesExactParameterNames() { + String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/test?YEARISDATETYPE=true" + + "&custom=characterEncoding=utf-8"; + + Assert.assertEquals(jdbcUrl + "&yearIsDateType=false&tinyInt1isBit=false" + + "&useUnicode=true&characterEncoding=utf-8", + StreamingJdbcUrlNormalizer.normalize(DataSourceType.MYSQL, jdbcUrl)); + } + + @Test + public void testNormalizeMysqlJdbcUrlIsIdempotent() { + String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/test?yearIsDateType=false" + + "&tinyInt1isBit=false&useUnicode=true&characterEncoding=utf-8"; + + Assert.assertEquals(jdbcUrl, + StreamingJdbcUrlNormalizer.normalize(DataSourceType.MYSQL, jdbcUrl)); + } + + @Test + public void testNormalizeOceanBaseJdbcUrlUsesMysqlRules() { + String jdbcUrl = StreamingJdbcUrlNormalizer.normalize( + DataSourceType.OCEANBASE, "jdbc:mysql://127.0.0.1:2883/test"); + + Assert.assertEquals("jdbc:mysql://127.0.0.1:2883/test?yearIsDateType=false" + + "&tinyInt1isBit=false&useUnicode=true&characterEncoding=utf-8", + jdbcUrl); + } + + @Test + public void testNormalizePostgresJdbcUrlDoesNotChange() { + String jdbcUrl = "jdbc:postgresql://127.0.0.1:5432/test"; + + Assert.assertEquals(jdbcUrl, + StreamingJdbcUrlNormalizer.normalize(DataSourceType.POSTGRES, jdbcUrl)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderOffsetTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderOffsetTest.java new file mode 100644 index 00000000000000..6efb9959748bcd --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProviderOffsetTest.java @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.offset.jdbc; + +import org.apache.doris.job.cdc.split.BinlogSplit; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; + +public class JdbcSourceOffsetProviderOffsetTest { + + @Test + public void testEndOffsetAdvancesWhenCurrentOffsetIsAhead() { + assertEndOffsetAdvancesWhenCurrentOffsetIsAhead(new TestJdbcSourceOffsetProvider(-1)); + } + + @Test + public void testTvfEndOffsetAdvancesWhenCurrentOffsetIsAhead() { + assertEndOffsetAdvancesWhenCurrentOffsetIsAhead(new TestJdbcTvfSourceOffsetProvider(-1)); + } + + @Test + public void testEndOffsetRemainsWhenItIsAheadOfCurrentOffset() { + JdbcSourceOffsetProvider provider = new TestJdbcSourceOffsetProvider(1); + Map currentOffset = Collections.singletonMap("lsn", "100"); + Map endOffset = Collections.singletonMap("lsn", "200"); + provider.setEndBinlogOffset(endOffset); + provider.setHasMoreData(false); + provider.updateOffset(new JdbcOffset( + Collections.singletonList(new BinlogSplit(currentOffset)))); + + Assert.assertTrue(provider.hasMoreDataToConsume()); + Assert.assertEquals(endOffset, provider.getEndBinlogOffset()); + } + + @Test + public void testStaleCompareDoesNotOverwriteRefreshedEndOffset() { + JdbcSourceOffsetProvider provider = new RefreshingEndOffsetProvider(); + provider.setEndBinlogOffset(Collections.singletonMap("lsn", "100")); + provider.updateOffset(new JdbcOffset( + Collections.singletonList(new BinlogSplit(Collections.singletonMap("lsn", "200"))))); + + Assert.assertTrue(provider.hasMoreDataToConsume()); + Assert.assertTrue(provider.hasMoreData); + Assert.assertEquals(Collections.singletonMap("lsn", "300"), provider.getEndBinlogOffset()); + } + + @Test + public void testStaleCompareDoesNotOverwriteAlteredCurrentOffsetState() { + JdbcSourceOffsetProvider provider = new AlteringCurrentOffsetProvider(); + provider.setEndBinlogOffset(Collections.singletonMap("lsn", "200")); + provider.updateOffset(new JdbcOffset( + Collections.singletonList(new BinlogSplit(Collections.singletonMap("lsn", "200"))))); + + Assert.assertTrue(provider.hasMoreDataToConsume()); + Assert.assertTrue(provider.hasMoreData); + Assert.assertEquals(Collections.singletonMap("lsn", "100"), + ((BinlogSplit) provider.currentOffset.getSplits().get(0)).getStartingOffset()); + } + + private static void assertEndOffsetAdvancesWhenCurrentOffsetIsAhead(JdbcSourceOffsetProvider provider) { + Map staleEndOffset = Collections.singletonMap("lsn", "100"); + Map committedOffset = Collections.singletonMap("lsn", "200"); + provider.setEndBinlogOffset(staleEndOffset); + provider.setHasMoreData(false); + + provider.updateOffset(new JdbcOffset( + Collections.singletonList(new BinlogSplit(committedOffset)))); + + Assert.assertEquals(staleEndOffset, provider.getEndBinlogOffset()); + Assert.assertFalse(provider.hasMoreDataToConsume()); + Assert.assertEquals(committedOffset, provider.getEndBinlogOffset()); + Assert.assertEquals("{\"lsn\":\"200\"}", provider.getShowMaxOffset()); + } + + private static class TestJdbcSourceOffsetProvider extends JdbcSourceOffsetProvider { + private final int compareResult; + + TestJdbcSourceOffsetProvider(int compareResult) { + this.compareResult = compareResult; + } + + @Override + protected int compareOffset(Map offsetFirst, Map offsetSecond) { + return compareResult; + } + } + + private static class TestJdbcTvfSourceOffsetProvider extends JdbcTvfSourceOffsetProvider { + private final int compareResult; + + TestJdbcTvfSourceOffsetProvider(int compareResult) { + this.compareResult = compareResult; + } + + @Override + protected int compareOffset(Map offsetFirst, Map offsetSecond) { + return compareResult; + } + } + + private static class RefreshingEndOffsetProvider extends JdbcSourceOffsetProvider { + @Override + protected int compareOffset(Map offsetFirst, Map offsetSecond) { + synchronized (splitsLock) { + endBinlogOffset = Collections.singletonMap("lsn", "300"); + hasMoreData = false; + } + return -1; + } + } + + private static class AlteringCurrentOffsetProvider extends JdbcSourceOffsetProvider { + @Override + protected int compareOffset(Map offsetFirst, Map offsetSecond) { + updateOffset(new JdbcOffset( + Collections.singletonList(new BinlogSplit(Collections.singletonMap("lsn", "100"))))); + return 0; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProviderTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProviderTest.java new file mode 100644 index 00000000000000..09f336fc4424a4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProviderTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.offset.jdbc; + +import org.apache.doris.job.cdc.DataSourceConfigKeys; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class JdbcTvfSourceOffsetProviderTest { + + @Test + public void testEnsureInitializedNormalizesMysqlJdbcUrl() throws Exception { + JdbcTvfSourceOffsetProvider provider = new JdbcTvfSourceOffsetProvider(); + Map properties = new HashMap<>(); + properties.put(DataSourceConfigKeys.TYPE, "mysql"); + properties.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://127.0.0.1:3306/test"); + properties.put(DataSourceConfigKeys.TABLE, "source_table"); + + provider.ensureInitialized(1L, properties); + + Assert.assertEquals("jdbc:mysql://127.0.0.1:3306/test?yearIsDateType=false" + + "&tinyInt1isBit=false&useUnicode=true&characterEncoding=utf-8", + provider.getSourceProperties().get(DataSourceConfigKeys.JDBC_URL)); + Assert.assertEquals("jdbc:mysql://127.0.0.1:3306/test", + properties.get(DataSourceConfigKeys.JDBC_URL)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/util/StreamingJobUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/util/StreamingJobUtilsTest.java index 3e090d4d3fe168..93c19074cd1a3d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/job/util/StreamingJobUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/job/util/StreamingJobUtilsTest.java @@ -21,6 +21,8 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.datasource.jdbc.client.JdbcClient; +import org.apache.doris.job.cdc.DataSourceConfigKeys; +import org.apache.doris.job.common.DataSourceType; import org.junit.Assert; import org.junit.Before; @@ -32,7 +34,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class StreamingJobUtilsTest { @@ -237,4 +241,13 @@ public void testGetColumnsWithVarcharPrimaryKeyLengthMultiplication() throws Exc Assert.assertNotNull(normalVarcharColumn); Assert.assertEquals(150, normalVarcharColumn.getType().getLength()); // 50 * 3 } + + @Test + public void testGetOceanBaseRemoteDbName() { + Map properties = new HashMap<>(); + properties.put(DataSourceConfigKeys.DATABASE, "test_db"); + + Assert.assertEquals("test_db", + StreamingJobUtils.getRemoteDbName(DataSourceType.OCEANBASE, properties)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/journal/bdbje/BDBEnvironmentTest.java b/fe/fe-core/src/test/java/org/apache/doris/journal/bdbje/BDBEnvironmentTest.java index a85f26b36158c3..729446fb4e016e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/journal/bdbje/BDBEnvironmentTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/journal/bdbje/BDBEnvironmentTest.java @@ -39,6 +39,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -59,6 +60,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; public class BDBEnvironmentTest { private static final Logger LOG = LogManager.getLogger(BDBEnvironmentTest.class); @@ -124,6 +126,19 @@ private byte[] randomBytes() { return byteArray; } + private void assertDatabaseValueEventually(BDBEnvironment environment, String dbName, + DatabaseEntry key, DatabaseEntry expectedValue) { + Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + Assertions.assertEquals(1, environment.getDatabaseNames().size()); + Database database = environment.openDatabase(dbName); + Assertions.assertNotNull(database); + DatabaseEntry actualValue = new DatabaseEntry(); + Assertions.assertEquals(OperationStatus.SUCCESS, + database.get(null, key, actualValue, LockMode.DEFAULT)); + Assertions.assertArrayEquals(expectedValue.getData(), actualValue.getData()); + }); + } + private static DatabaseEntry longToEntry(long value) { DatabaseEntry key = new DatabaseEntry(); TupleBinding idBinding = TupleBinding.getPrimitiveBinding(Long.class); @@ -690,13 +705,13 @@ public void testReadTxnIsNotMatched() throws Exception { continue; } - Assertions.assertEquals(1, entryPair.first.getDatabaseNames().size()); - Database followerDb = entryPair.first.openDatabase(beginDbName); - DatabaseEntry readValue = new DatabaseEntry(); - Assertions.assertEquals(OperationStatus.SUCCESS, followerDb.get(null, key, readValue, LockMode.DEFAULT)); - Assertions.assertEquals(new String(value.getData()), new String(readValue.getData())); + assertDatabaseValueEventually(entryPair.first, beginDbName, key, value); } + long count = masterDb.count(); + DatabaseEntry localCommitKey = new DatabaseEntry(new byte[]{1, 2, 3}); + DatabaseEntry localCommitValue = new DatabaseEntry(new byte[]{4, 5, 6}); + ReplicatedEnvironment replicatedEnvironment = masterPair.first.getReplicatedEnvironment(); Field envImplField = ReplicatedEnvironment.class.getDeclaredField("repEnvironmentImpl"); envImplField.setAccessible(true); @@ -706,19 +721,20 @@ public void testReadTxnIsNotMatched() throws Exception { Field environmentImplField = com.sleepycat.je.Environment.class.getDeclaredField("environmentImpl"); environmentImplField.setAccessible(true); RepImpl implSpy = Mockito.spy(impl); - envImplField.set(replicatedEnvironment, implSpy); - environmentImplField.set(replicatedEnvironment, implSpy); Mockito.doThrow(new InsufficientAcksException("mocked")) .when(implSpy).postLogCommitHook(Mockito.any(), Mockito.any()); - long count = masterDb.count(); - final Database oldMasterDb = masterDb; - Assertions.assertThrows(InsufficientAcksException.class, () -> { - // Since this key is not replicated to any replicas, it should not be read. - DatabaseEntry k = new DatabaseEntry(new byte[]{1, 2, 3}); - DatabaseEntry v = new DatabaseEntry(new byte[]{4, 5, 6}); - oldMasterDb.put(null, k, v); - }); + try { + envImplField.set(replicatedEnvironment, implSpy); + environmentImplField.set(replicatedEnvironment, implSpy); + final Database oldMasterDb = masterDb; + // Simulate an acknowledgement failure after the transaction is committed locally. + Assertions.assertThrows(InsufficientAcksException.class, + () -> oldMasterDb.put(null, localCommitKey, localCommitValue)); + } finally { + envImplField.set(replicatedEnvironment, impl); + environmentImplField.set(replicatedEnvironment, impl); + } LOG.info("close old master {} | {}", masterPair.second.name, masterPair.second.dir); masterDb.close(); @@ -741,9 +757,10 @@ public void testReadTxnIsNotMatched() throws Exception { // The local commit txn is readable!!! Assertions.assertEquals(count + 1, masterDb.count()); - key = new DatabaseEntry(new byte[]{1, 2, 3}); DatabaseEntry readValue = new DatabaseEntry(); - Assertions.assertEquals(OperationStatus.SUCCESS, masterDb.get(null, key, readValue, LockMode.DEFAULT)); + Assertions.assertEquals(OperationStatus.SUCCESS, + masterDb.get(null, localCommitKey, readValue, LockMode.DEFAULT)); + Assertions.assertArrayEquals(localCommitValue.getData(), readValue.getData()); } finally { followersInfo.stream().forEach(entryPair -> { entryPair.first.close(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/StreamLoadHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/StreamLoadHandlerTest.java index 20ecd88538c503..8f266e81ab82e7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/StreamLoadHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/StreamLoadHandlerTest.java @@ -17,14 +17,25 @@ package org.apache.doris.load; +import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TStreamLoadPutRequest; +import org.apache.doris.thrift.TStreamLoadPutResult; import org.junit.Assert; import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.util.Arrays; import java.util.List; @@ -48,6 +59,92 @@ public void testSelectBackendSkipsDecommissioningBackend() throws Exception { } } + @Test + public void testSetCloudClusterUsesBackendComputeGroup() throws Exception { + SystemInfoService originalSystemInfoService = Env.getCurrentSystemInfo(); + String originalCloudUniqueId = Config.cloud_unique_id; + Backend backend = createBackend(10001L, "127.0.0.1"); + backend.setCloudClusterName("backend_compute_group"); + CloudSystemInfoService systemInfoService = + new TestCloudSystemInfoService(Arrays.asList(backend)); + TStreamLoadPutRequest request = new TStreamLoadPutRequest(); + request.setUser(""); + request.setBackendId(backend.getId()); + request.setCloudCluster("header_compute_group"); + + try { + Config.cloud_unique_id = "test_cloud_unique_id"; + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", systemInfoService); + ConnectContext.remove(); + + StreamLoadHandler handler = new StreamLoadHandler( + request, null, new TStreamLoadPutResult(), "127.0.0.1"); + handler.setCloudCluster(); + + Assert.assertEquals("backend_compute_group", + ConnectContext.get().getSessionVariable().getCloudCluster()); + Assert.assertEquals("backend_compute_group", request.getCloudCluster()); + } finally { + ConnectContext.remove(); + Config.cloud_unique_id = originalCloudUniqueId; + Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", originalSystemInfoService); + } + } + + @Test + public void testGroupCommitValidatesBackendComputeGroupPrivilege() throws Exception { + String originalCloudUniqueId = Config.cloud_unique_id; + Backend backend = createBackend(10001L, "127.0.0.1"); + backend.setCloudClusterName("backend_compute_group"); + CloudSystemInfoService systemInfoService = + new TestCloudSystemInfoService(Arrays.asList(backend)); + CloudEnv cloudEnv = Mockito.mock(CloudEnv.class); + InternalCatalog internalCatalog = Mockito.mock(InternalCatalog.class); + Auth auth = Mockito.mock(Auth.class); + Mockito.when(cloudEnv.getInternalCatalog()).thenReturn(internalCatalog); + Mockito.when(internalCatalog.getName()).thenReturn(InternalCatalog.INTERNAL_CATALOG_NAME); + Mockito.when(cloudEnv.getAuth()).thenReturn(auth); + Mockito.doAnswer(invocation -> { + List currentUser = invocation.getArgument(3); + currentUser.add(UserIdentity.createAnalyzedUserIdentWithIp("test_user", "%")); + return null; + }).when(auth).checkPlainPassword(Mockito.eq("test_user"), Mockito.anyString(), + Mockito.anyString(), Mockito.anyList()); + Mockito.doThrow(new DdlException("USAGE denied")) + .when(cloudEnv).changeCloudCluster( + Mockito.eq("backend_compute_group"), Mockito.any(ConnectContext.class)); + + TStreamLoadPutRequest request = new TStreamLoadPutRequest(); + request.setUser("test_user"); + request.setUserIp("127.0.0.1"); + request.setPasswd("test_password"); + request.setBackendId(backend.getId()); + request.setCloudCluster("header_compute_group"); + request.setGroupCommitMode("sync_mode"); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + Config.cloud_unique_id = "test_cloud_unique_id"; + mockedEnv.when(Env::getCurrentEnv).thenReturn(cloudEnv); + mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfoService); + ConnectContext.remove(); + + StreamLoadHandler handler = new StreamLoadHandler( + request, null, new TStreamLoadPutResult(), "127.0.0.1"); + try { + handler.setCloudCluster(); + Assert.fail("group commit should validate compute group privilege"); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("USAGE denied")); + } + + Mockito.verify(cloudEnv).changeCloudCluster( + Mockito.eq("backend_compute_group"), Mockito.any(ConnectContext.class)); + } finally { + ConnectContext.remove(); + Config.cloud_unique_id = originalCloudUniqueId; + } + } + private Backend createBackend(long id, String host) { Backend backend = new Backend(id, host, 9050); backend.setAlive(true); @@ -65,5 +162,10 @@ private TestCloudSystemInfoService(List backends) { public List getBackendsByClusterName(final String clusterName) { return backends; } + + @Override + public Backend getBackend(long backendId) { + return backends.stream().filter(backend -> backend.getId() == backendId).findFirst().orElse(null); + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java index 8876c6a8aea9bc..263515975437fa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java @@ -21,6 +21,8 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Table; +import org.apache.doris.cloud.transaction.TxnUtil; +import org.apache.doris.common.Config; import org.apache.doris.common.InternalErrorCode; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; @@ -33,6 +35,9 @@ import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.persist.EditLog; import org.apache.doris.thrift.TKafkaRLTaskProgress; +import org.apache.doris.thrift.TLoadSourceType; +import org.apache.doris.thrift.TRLTaskTxnCommitAttachment; +import org.apache.doris.thrift.TUniqueId; import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.GlobalTransactionMgrIface; import org.apache.doris.transaction.TransactionException; @@ -53,6 +58,27 @@ import java.util.Map; public class RoutineLoadJobTest { + @Test + public void testFirstErrorMsgInTxnCommitAttachment() { + String overlongFirstErrorMsg = Strings.repeat("x", Config.first_error_msg_max_length + 1); + TRLTaskTxnCommitAttachment thriftAttachment = new TRLTaskTxnCommitAttachment(); + thriftAttachment.setLoadSourceType(TLoadSourceType.KAFKA); + thriftAttachment.setId(new TUniqueId(1, 2)); + thriftAttachment.setJobId(3); + thriftAttachment.setKafkaRLTaskProgress(new TKafkaRLTaskProgress(Maps.newHashMap())); + thriftAttachment.setErrorLogUrl("http://127.0.0.1/error_log"); + thriftAttachment.setFirstErrorMsg(overlongFirstErrorMsg); + + RLTaskTxnCommitAttachment attachment = new RLTaskTxnCommitAttachment(thriftAttachment); + Assert.assertEquals(Config.first_error_msg_max_length, attachment.getFirstErrorMsg().length()); + Assert.assertTrue(attachment.getFirstErrorMsg().endsWith("...")); + + RLTaskTxnCommitAttachment cloudAttachment = TxnUtil.rtTaskTxnCommitAttachmentFromPb( + TxnUtil.rlTaskTxnCommitAttachmentToPb(attachment)); + Assert.assertEquals("http://127.0.0.1/error_log", cloudAttachment.getErrorLogUrl()); + Assert.assertEquals(attachment.getFirstErrorMsg(), cloudAttachment.getFirstErrorMsg()); + } + @Test public void testAfterAbortedReasonOffsetOutOfRange() throws UserException { Env env = Mockito.mock(Env.class); @@ -95,6 +121,8 @@ public void testAfterAborted() throws UserException { tKafkaRLTaskProgress.partitionCmtOffset = Maps.newHashMap(); KafkaProgress kafkaProgress = new KafkaProgress(tKafkaRLTaskProgress); Deencapsulation.setField(attachment, "progress", kafkaProgress); + Deencapsulation.setField(attachment, "errorLogUrl", "http://127.0.0.1/error_log"); + Deencapsulation.setField(attachment, "firstErrorMsg", "invalid source row"); KafkaProgress currentProgress = new KafkaProgress(tKafkaRLTaskProgress); @@ -114,6 +142,8 @@ public void testAfterAborted() throws UserException { Assert.assertEquals(RoutineLoadJob.JobState.RUNNING, routineLoadJob.getState()); Assert.assertEquals(new Long(1), Deencapsulation.getField(jobStatistic, "abortedTaskNum")); + Assert.assertEquals("http://127.0.0.1/error_log", routineLoadJob.getErrorLogUrls().peek()); + Assert.assertEquals("invalid source row", routineLoadJob.getFirstErrorMsg()); } @Test @@ -153,10 +183,13 @@ public void testGetShowInfo() { Deencapsulation.setField(routineLoadJob, "pauseReason", errorReason); Deencapsulation.setField(routineLoadJob, "progress", kafkaProgress); Deencapsulation.setField(routineLoadJob, "userIdentity", userIdentity); + Deencapsulation.setField(routineLoadJob, "firstErrorMsg", "invalid source row"); List showInfo = routineLoadJob.getShowInfo(); Assert.assertEquals(true, showInfo.stream().filter(entity -> !Strings.isNullOrEmpty(entity)) .anyMatch(entity -> entity.equals(errorReason.toString()))); + Assert.assertEquals(24, showInfo.size()); + Assert.assertEquals("invalid source row", showInfo.get(23)); } @Test @@ -276,11 +309,17 @@ public void testUpdateTotalMoreThanBatch() { RoutineLoadStatistic jobStatistic = Deencapsulation.getField(routineLoadJob, "jobStatistic"); Deencapsulation.setField(jobStatistic, "currentErrorRows", 1); Deencapsulation.setField(jobStatistic, "currentTotalRows", 99); + Deencapsulation.setField(routineLoadJob, "otherMsg", "previous warning"); + routineLoadJob.getErrorLogUrls().add("http://127.0.0.1/error_log"); + Deencapsulation.setField(routineLoadJob, "firstErrorMsg", "invalid source row"); Deencapsulation.invoke(routineLoadJob, "updateNumOfData", 2L, 0L, 0L, 1L, 1L, false); Assert.assertEquals(RoutineLoadJob.JobState.RUNNING, Deencapsulation.getField(routineLoadJob, "state")); Assert.assertEquals(new Long(0), Deencapsulation.getField(jobStatistic, "currentErrorRows")); Assert.assertEquals(new Long(0), Deencapsulation.getField(jobStatistic, "currentTotalRows")); + Assert.assertEquals("", Deencapsulation.getField(routineLoadJob, "otherMsg")); + Assert.assertTrue(routineLoadJob.getErrorLogUrls().isEmpty()); + Assert.assertEquals("", routineLoadJob.getFirstErrorMsg()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/CloudAuthTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/CloudAuthTest.java index 31880402db5918..1c8d739aeac2d1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/CloudAuthTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/CloudAuthTest.java @@ -24,8 +24,8 @@ import org.apache.doris.catalog.AccessPrivilege; import org.apache.doris.catalog.AccessPrivilegeWithCols; import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.catalog.CloudComputeGroupMeta; import org.apache.doris.cloud.catalog.CloudEnv; -import org.apache.doris.cloud.catalog.ComputeGroup; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; @@ -336,12 +336,12 @@ public void testVirtualComputeGroup() throws Exception { Assert.assertTrue(accessManager.checkCloudPriv(new UserIdentity("testUser", "%"), "vcg", PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); // create vcg, sub cg(cg1, cg2), add to systemInfoService - ComputeGroup vcg = new ComputeGroup("vcg_id", "vcg", ComputeGroup.ComputeTypeEnum.VIRTUAL); + CloudComputeGroupMeta vcg = new CloudComputeGroupMeta("vcg_id", "vcg", CloudComputeGroupMeta.ComputeTypeEnum.VIRTUAL); vcg.setSubComputeGroups(Lists.newArrayList("cg2", "cg1")); systemInfoService.addComputeGroup("vcg_id", vcg); - ComputeGroup cg = new ComputeGroup("vcg_id", "vcg", ComputeGroup.ComputeTypeEnum.COMPUTE); + CloudComputeGroupMeta cg = new CloudComputeGroupMeta("vcg_id", "vcg", CloudComputeGroupMeta.ComputeTypeEnum.COMPUTE); systemInfoService.addComputeGroup("cg", cg); - ComputeGroup.Policy policy = new ComputeGroup.Policy(); + CloudComputeGroupMeta.Policy policy = new CloudComputeGroupMeta.Policy(); policy.setActiveComputeGroup("cg1"); policy.setStandbyComputeGroup("cg2"); vcg.setPolicy(policy); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/RangerDorisAccessControllerFactoryTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/RangerDorisAccessControllerFactoryTest.java new file mode 100644 index 00000000000000..7029f8e4389f8e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/RangerDorisAccessControllerFactoryTest.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mysql.privilege; + +import org.apache.doris.catalog.authorizer.ranger.doris.RangerDorisAccessController; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedConstruction; +import org.mockito.Mockito; + +import java.util.Collections; + +public class RangerDorisAccessControllerFactoryTest { + @Test + public void testCreateAccessControllerReturnsSingleton() { + try (MockedConstruction mockedConstruction = + Mockito.mockConstruction(RangerDorisAccessController.class)) { + RangerDorisAccessController first = new RangerDorisAccessControllerFactory() + .createAccessController(Collections.emptyMap()); + RangerDorisAccessController second = new RangerDorisAccessControllerFactory() + .createAccessController(Collections.emptyMap()); + + Assert.assertEquals(1, mockedConstruction.constructed().size()); + Assert.assertSame(first, second); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java index 34de78c7a76318..58e32440364c49 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java @@ -17,9 +17,14 @@ package org.apache.doris.nereids.glue.translator; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ListPartitionItem; @@ -29,9 +34,13 @@ import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.IntegerType; @@ -45,6 +54,7 @@ import org.mockito.Mockito; import java.util.Map; +import java.util.function.Function; class RuntimeFilterPartitionPruneClassifierTest { @Test @@ -94,8 +104,32 @@ void testInOrBloomRangePartitionStillSupported() { assertSupportedIncreasingPartitions(classification); } + @Test + void testRejectNoneMovableListTargetExpression() { + RuntimeFilterPartitionPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM, + targetSlot -> new FunctionCallExpr("assert_true", ImmutableList.of( + new BinaryPredicate(BinaryPredicate.Operator.NE, targetSlot, new IntLiteral(0)), + new StringLiteral("rfpp_expr_in_only_error")), false), + targetSlot -> new AssertTrue( + new GreaterThan(targetSlot, new IntegerLiteral(0)), + new VarcharLiteral("rfpp_expr_in_only_error"))); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getUnsupportedReason().contains("non-movable")); + Assertions.assertTrue(classification.getPartitionMonotonicity().isEmpty()); + } + private RuntimeFilterPartitionPruneClassifier.Classification classify( TRuntimeFilterType filterType, PartitionType partitionType, PartitionItem partitionItem) { + return classify(filterType, partitionType, partitionItem, targetSlot -> targetSlot, + targetSlot -> targetSlot); + } + + private RuntimeFilterPartitionPruneClassifier.Classification classify( + TRuntimeFilterType filterType, PartitionType partitionType, PartitionItem partitionItem, + Function legacyTargetFactory, + Function nereidsTargetFactory) { Column partitionColumn = new Column("part_col", PrimitiveType.INT); SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(1), new TupleId(1)); slotDescriptor.setColumn(partitionColumn); @@ -114,7 +148,8 @@ private RuntimeFilterPartitionPruneClassifier.Classification classify( Mockito.when(partitionInfo.getItem(1L)).thenReturn(partitionItem); Mockito.when(partitionInfo.getItem(2L)).thenReturn(partitionItem); - return RuntimeFilterPartitionPruneClassifier.classify(filterType, targetSlot, nereidsTarget, scanNode); + return RuntimeFilterPartitionPruneClassifier.classify(filterType, + legacyTargetFactory.apply(targetSlot), nereidsTargetFactory.apply(nereidsTarget), scanNode); } private void assertSupportedIncreasingPartitions( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/hint/LeadingHintTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/hint/LeadingHintTest.java deleted file mode 100644 index 00f0c45437f091..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/hint/LeadingHintTest.java +++ /dev/null @@ -1,219 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.hint; - -import org.apache.doris.common.Pair; -import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; -import org.apache.doris.nereids.trees.plans.JoinType; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class LeadingHintTest { - - @Test - public void testLeftSemiJoinConstrainedSideMatchesExactly() { - LeadingHint leading = new LeadingHint("Leading"); - long leftHand = LongBitmap.newBitmap(0); - long rightHand = LongBitmap.newBitmap(1); - long extraTable = LongBitmap.newBitmap(2); - JoinConstraint semiJoinConstraint = addJoinConstraint(leading, leftHand, rightHand, - JoinType.LEFT_SEMI_JOIN); - - Pair exactReversed = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(leftHand, rightHand), rightHand, leftHand); - assertMatchedJoinConstraint(semiJoinConstraint, exactReversed, true); - - Pair withoutRetainedSide = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(rightHand, extraTable), rightHand, extraTable); - assertNoMatchedJoinConstraint(withoutRetainedSide); - - Pair withExtraTable = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(leftHand, rightHand, extraTable), - LongBitmap.newBitmapUnion(rightHand, extraTable), - leftHand); - assertNoMatchedJoinConstraint(withExtraTable); - } - - @Test - public void testLeftAntiJoinConstrainedSideMatchesExactly() { - LeadingHint leading = new LeadingHint("Leading"); - long leftHand = LongBitmap.newBitmap(0); - long rightHand = LongBitmap.newBitmap(1); - long extraTable = LongBitmap.newBitmap(2); - JoinConstraint antiJoinConstraint = addJoinConstraint(leading, leftHand, rightHand, - JoinType.LEFT_ANTI_JOIN); - - Pair exactReversed = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(leftHand, rightHand), rightHand, leftHand); - assertMatchedJoinConstraint(antiJoinConstraint, exactReversed, true); - - Pair withExtraTable = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(leftHand, rightHand, extraTable), - LongBitmap.newBitmapUnion(rightHand, extraTable), - leftHand); - assertNoMatchedJoinConstraint(withExtraTable); - } - - @Test - public void testRightSemiJoinConstrainedSideMatchesExactly() { - LeadingHint leading = new LeadingHint("Leading"); - long leftHand = LongBitmap.newBitmap(0); - long rightHand = LongBitmap.newBitmap(1); - long extraTable = LongBitmap.newBitmap(2); - JoinConstraint semiJoinConstraint = addJoinConstraint(leading, leftHand, rightHand, - JoinType.RIGHT_SEMI_JOIN); - - Pair exact = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(leftHand, rightHand), leftHand, rightHand); - assertMatchedJoinConstraint(semiJoinConstraint, exact, false); - - Pair withoutConstrainedSide = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(rightHand, extraTable), rightHand, extraTable); - assertNoMatchedJoinConstraint(withoutConstrainedSide); - - Pair withExtraTable = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(leftHand, rightHand, extraTable), - LongBitmap.newBitmapUnion(leftHand, extraTable), - rightHand); - assertNoMatchedJoinConstraint(withExtraTable); - } - - @Test - public void testRightAntiJoinConstrainedSideMatchesExactly() { - LeadingHint leading = new LeadingHint("Leading"); - long leftHand = LongBitmap.newBitmap(0); - long rightHand = LongBitmap.newBitmap(1); - long extraTable = LongBitmap.newBitmap(2); - JoinConstraint antiJoinConstraint = addJoinConstraint(leading, leftHand, rightHand, - JoinType.RIGHT_ANTI_JOIN); - - Pair exact = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(leftHand, rightHand), leftHand, rightHand); - assertMatchedJoinConstraint(antiJoinConstraint, exact, false); - - Pair withExtraTable = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(leftHand, rightHand, extraTable), - LongBitmap.newBitmapUnion(leftHand, extraTable), - rightHand); - assertNoMatchedJoinConstraint(withExtraTable); - } - - @Test - public void testCompositeLeftSemiAndAntiJoinConstrainedSideCanNotBeSplit() { - assertCompositeConstrainedSideCanNotBeSplit(JoinType.LEFT_SEMI_JOIN); - assertCompositeConstrainedSideCanNotBeSplit(JoinType.LEFT_ANTI_JOIN); - } - - @Test - public void testCompositeRightSemiAndAntiJoinConstrainedSideCanNotBeSplit() { - assertCompositeConstrainedSideCanNotBeSplit(JoinType.RIGHT_SEMI_JOIN); - assertCompositeConstrainedSideCanNotBeSplit(JoinType.RIGHT_ANTI_JOIN); - } - - @Test - public void testCompositeLeftSemiAndAntiJoinRetainedSideCanUseMinHand() { - assertCompositeRetainedSideCanUseMinHand(JoinType.LEFT_SEMI_JOIN); - assertCompositeRetainedSideCanUseMinHand(JoinType.LEFT_ANTI_JOIN); - } - - @Test - public void testCompositeRightSemiAndAntiJoinRetainedSideCanUseMinHand() { - assertCompositeRetainedSideCanUseMinHand(JoinType.RIGHT_SEMI_JOIN); - assertCompositeRetainedSideCanUseMinHand(JoinType.RIGHT_ANTI_JOIN); - } - - private JoinConstraint addJoinConstraint(LeadingHint leading, long leftHand, long rightHand, - JoinType joinType) { - JoinConstraint joinConstraint = new JoinConstraint(leftHand, rightHand, leftHand, rightHand, joinType, true); - leading.getJoinConstraintList().add(joinConstraint); - return joinConstraint; - } - - private JoinConstraint addJoinConstraint(LeadingHint leading, long minLeftHand, long minRightHand, - long leftHand, long rightHand, JoinType joinType) { - JoinConstraint joinConstraint = new JoinConstraint(minLeftHand, minRightHand, leftHand, rightHand, - joinType, true); - leading.getJoinConstraintList().add(joinConstraint); - return joinConstraint; - } - - private void assertCompositeConstrainedSideCanNotBeSplit(JoinType joinType) { - LeadingHint leading = new LeadingHint("Leading"); - long leftHand = LongBitmap.newBitmap(0); - long rightHand = LongBitmap.newBitmap(1); - long extraTable = LongBitmap.newBitmap(2); - long joinTable = LongBitmap.newBitmapUnion(leftHand, rightHand, extraTable); - - if (joinType.isRightSemiOrAntiJoin()) { - JoinConstraint joinConstraint = addJoinConstraint(leading, leftHand, rightHand, - LongBitmap.newBitmapUnion(leftHand, extraTable), rightHand, joinType); - - Pair exact = leading.getJoinConstraint( - joinTable, LongBitmap.newBitmapUnion(leftHand, extraTable), rightHand); - assertMatchedJoinConstraint(joinConstraint, exact, false); - - Pair splitConstrainedSide = leading.getJoinConstraint( - joinTable, leftHand, LongBitmap.newBitmapUnion(rightHand, extraTable)); - assertNoMatchedJoinConstraint(splitConstrainedSide); - } else { - JoinConstraint joinConstraint = addJoinConstraint(leading, leftHand, rightHand, - leftHand, LongBitmap.newBitmapUnion(rightHand, extraTable), joinType); - - Pair exact = leading.getJoinConstraint( - joinTable, leftHand, LongBitmap.newBitmapUnion(rightHand, extraTable)); - assertMatchedJoinConstraint(joinConstraint, exact, false); - - Pair splitConstrainedSide = leading.getJoinConstraint( - joinTable, LongBitmap.newBitmapUnion(leftHand, extraTable), rightHand); - assertNoMatchedJoinConstraint(splitConstrainedSide); - } - } - - private void assertCompositeRetainedSideCanUseMinHand(JoinType joinType) { - LeadingHint leading = new LeadingHint("Leading"); - long leftHand = LongBitmap.newBitmap(0); - long rightHand = LongBitmap.newBitmap(1); - long extraTable = LongBitmap.newBitmap(2); - - JoinConstraint joinConstraint; - if (joinType.isRightSemiOrAntiJoin()) { - joinConstraint = addJoinConstraint(leading, leftHand, rightHand, - leftHand, LongBitmap.newBitmapUnion(rightHand, extraTable), joinType); - } else { - joinConstraint = addJoinConstraint(leading, leftHand, rightHand, - LongBitmap.newBitmapUnion(leftHand, extraTable), rightHand, joinType); - } - - Pair minRetainedSide = leading.getJoinConstraint( - LongBitmap.newBitmapUnion(leftHand, rightHand), leftHand, rightHand); - assertMatchedJoinConstraint(joinConstraint, minRetainedSide, false); - } - - private void assertMatchedJoinConstraint(JoinConstraint expected, Pair actual, - boolean reversed) { - Assertions.assertSame(expected, actual.first); - Assertions.assertTrue(actual.second); - Assertions.assertEquals(reversed, actual.first.isReversed()); - } - - private void assertNoMatchedJoinConstraint(Pair actual) { - Assertions.assertNull(actual.first); - Assertions.assertTrue(actual.second); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java new file mode 100644 index 00000000000000..3b3355c06079e1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/IcebergNestedSchemaEvolutionParserTest.java @@ -0,0 +1,462 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.analysis.ColumnPath; +import org.apache.doris.nereids.exceptions.ParseException; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand; +import org.apache.doris.nereids.trees.plans.commands.info.AddColumnOp; +import org.apache.doris.nereids.trees.plans.commands.info.AddColumnsOp; +import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp; +import org.apache.doris.nereids.trees.plans.commands.info.DropColumnOp; +import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnCommentOp; +import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnOp; +import org.apache.doris.nereids.trees.plans.commands.info.RenameColumnOp; +import org.apache.doris.nereids.types.StructType; +import org.apache.doris.qe.SqlModeHelper; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +public class IcebergNestedSchemaEvolutionParserTest { + + private final NereidsParser parser = new NereidsParser(); + + @Test + public void testParseNestedAddColumnPaths() { + assertSingleClausePath("ALTER TABLE t ADD COLUMN s.c STRING NULL", + AddColumnOp.class, "s.c"); + assertSingleClausePath("ALTER TABLE t ADD COLUMN s.first_col INT NULL FIRST", + AddColumnOp.class, "s.first_col"); + assertSingleClausePath("ALTER TABLE t ADD COLUMN s.after_col INT NULL AFTER a", + AddColumnOp.class, "s.after_col"); + assertSingleClausePath("ALTER TABLE t ADD COLUMN arr.element.y INT NULL", + AddColumnOp.class, "arr.element.y"); + assertSingleClausePath("ALTER TABLE t ADD COLUMN m.value.y INT NULL", + AddColumnOp.class, "m.value.y"); + } + + @Test + public void testParseNestedModifyDropRenamePaths() { + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN s.a BIGINT", + ModifyColumnOp.class, "s.a"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN s.a BIGINT AFTER b", + ModifyColumnOp.class, "s.a"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN arr.element BIGINT", + ModifyColumnOp.class, "arr.element"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN m.value BIGINT", + ModifyColumnOp.class, "m.value"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN s.a COMMENT 'nested comment'", + ModifyColumnCommentOp.class, "s.a"); + assertSingleClausePath("ALTER TABLE t DROP COLUMN s.c", + DropColumnOp.class, "s.c"); + assertSingleClausePath("ALTER TABLE t DROP COLUMN arr.element.y", + DropColumnOp.class, "arr.element.y"); + assertSingleClausePath("ALTER TABLE t DROP COLUMN m.value.y", + DropColumnOp.class, "m.value.y"); + assertSingleClausePath("ALTER TABLE t RENAME COLUMN s.c TO c2", + RenameColumnOp.class, "s.c"); + assertSingleClausePath("ALTER TABLE t RENAME COLUMN arr.element.y TO y2", + RenameColumnOp.class, "arr.element.y"); + assertSingleClausePath("ALTER TABLE t RENAME COLUMN m.value.y TO y2", + RenameColumnOp.class, "m.value.y"); + } + + @Test + public void testNestedColumnDefaultClausesAreNotInGrammar() { + for (String sql : List.of( + "ALTER TABLE t ADD COLUMN s.b BIGINT NULL DEFAULT 7", + "ALTER TABLE t ADD COLUMN s.c BIGINT NULL DEFAULT NULL", + "ALTER TABLE t ADD COLUMN s.ts DATETIME NULL DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP", + "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT 7", + "ALTER TABLE t MODIFY COLUMN s.a BIGINT DEFAULT NULL", + "ALTER TABLE t MODIFY COLUMN s.ts DATETIME DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP")) { + Assertions.assertThrows(ParseException.class, () -> parser.parseSingle(sql), sql); + } + } + + @Test + public void testTopLevelColumnKeepsExistingDefaultGrammar() { + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN b BIGINT NULL DEFAULT 7", AddColumnOp.class, "b"); + Assertions.assertTrue(add.getColumnDef().hasDefaultValue()); + } + + @Test + public void testTopLevelColumnRoundTripPreservesOmittedIntent() { + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN added INT", AddColumnOp.class, "added"); + String renderedAdd = add.toSql(); + Assertions.assertFalse(renderedAdd.contains(" NULL")); + Assertions.assertFalse(renderedAdd.contains(" COMMENT ")); + AddColumnOp reparsedAdd = assertSingleClausePath( + "ALTER TABLE t " + renderedAdd, AddColumnOp.class, "added"); + Assertions.assertFalse(reparsedAdd.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertFalse(reparsedAdd.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN existing BIGINT", ModifyColumnOp.class, "existing"); + String renderedModify = modify.toSql(); + Assertions.assertFalse(renderedModify.contains(" NULL")); + Assertions.assertFalse(renderedModify.contains(" COMMENT ")); + ModifyColumnOp reparsedModify = assertSingleClausePath( + "ALTER TABLE t " + renderedModify, ModifyColumnOp.class, "existing"); + Assertions.assertFalse(reparsedModify.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertFalse(reparsedModify.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + } + + @Test + public void testLegacyStringConstructorsKeepDottedTopLevelNames() { + DropColumnOp drop = new DropColumnOp("top.level", null, Collections.emptyMap()); + RenameColumnOp rename = new RenameColumnOp("top.level", "renamed"); + ModifyColumnCommentOp comment = new ModifyColumnCommentOp("top.level", "comment"); + + Assertions.assertFalse(drop.getColumnPath().isNested()); + Assertions.assertFalse(rename.getColumnPath().isNested()); + Assertions.assertFalse(comment.getColumnPath().isNested()); + Assertions.assertEquals("top.level", drop.getColumnPath().getFullPath()); + Assertions.assertEquals("top.level", rename.getColumnPath().getFullPath()); + Assertions.assertEquals("top.level", comment.getColumnPath().getFullPath()); + } + + @Test + public void testQuotedNestedIdentifiersAreNormalized() { + ModifyColumnOp dottedTopLevel = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN `a.b` BIGINT", + ModifyColumnOp.class, "a.b"); + Assertions.assertFalse(dottedTopLevel.getColumnPath().isNested()); + + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN info.`Metric` BIGINT", + ModifyColumnOp.class, "info.Metric"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN m_scalar.`key` BIGINT", + ModifyColumnOp.class, "m_scalar.key"); + assertSingleClausePath("ALTER TABLE t MODIFY COLUMN info.`Metric``Name` COMMENT 'quoted'", + ModifyColumnCommentOp.class, "info.Metric`Name"); + + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN info.`New``Field` INT NULL AFTER `Old``Field`", + AddColumnOp.class, "info.New`Field"); + Assertions.assertEquals("Old`Field", add.getColPos().getLastCol()); + AddColumnOp reparsedAdd = assertSingleClausePath( + "ALTER TABLE t " + add.toSql(), AddColumnOp.class, "info.New`Field"); + Assertions.assertEquals("Old`Field", reparsedAdd.getColPos().getLastCol()); + + RenameColumnOp rename = assertSingleClausePath( + "ALTER TABLE t RENAME COLUMN info.`Metric``Name` TO `New``Metric`", + RenameColumnOp.class, "info.Metric`Name"); + Assertions.assertEquals("New`Metric", rename.getNewColName()); + RenameColumnOp reparsedRename = assertSingleClausePath( + "ALTER TABLE t " + rename.toSql(), RenameColumnOp.class, "info.Metric`Name"); + Assertions.assertEquals("New`Metric", reparsedRename.getNewColName()); + } + + @Test + public void testStructMemberIdentifiersRoundTrip() { + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN info.payload " + + "STRUCT<`key`:INT,`Metric``Name`:STRING> NULL", + AddColumnOp.class, "info.payload"); + assertStructMemberNames((StructType) add.getColumnDef().getType()); + Assertions.assertTrue(add.toSql().contains("STRUCT<`key`:INT,`metric``name`:TEXT>")); + AddColumnOp reparsedAdd = assertSingleClausePath( + "ALTER TABLE t " + add.toSql(), AddColumnOp.class, "info.payload"); + assertStructMemberNames((StructType) reparsedAdd.getColumnDef().getType()); + + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.payload " + + "STRUCT<`key`:INT,`Metric``Name`:STRING>", + ModifyColumnOp.class, "info.payload"); + assertStructMemberNames((StructType) modify.getColumnDef().getType()); + ModifyColumnOp reparsedModify = assertSingleClausePath( + "ALTER TABLE t " + modify.toSql(), ModifyColumnOp.class, "info.payload"); + assertStructMemberNames((StructType) reparsedModify.getColumnDef().getType()); + + ModifyColumnOp ordinary = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.payload STRUCT", + ModifyColumnOp.class, "info.payload"); + Assertions.assertTrue(ordinary.toSql().contains("STRUCT")); + } + + @Test + public void testStructFieldSqlRoundTripIsLocaleIndependent() { + Locale originalLocale = Locale.getDefault(); + try { + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + StructType structType = (StructType) parser.parseDataType("STRUCT<`in`:INT>"); + String renderedType = structType.toSql(); + + Assertions.assertTrue(renderedType.contains("`in`:INT")); + Assertions.assertDoesNotThrow(() -> parser.parseDataType(renderedType)); + } finally { + Locale.setDefault(originalLocale); + } + } + + @Test + public void testEmptyQuotedIdentifiersAreRejectedAsParseErrors() { + for (String sql : List.of( + "ALTER TABLE t ADD COLUMN `` INT NULL", + "ALTER TABLE t ADD COLUMN info.`` INT NULL", + "ALTER TABLE t MODIFY COLUMN info.`` BIGINT", + "ALTER TABLE t MODIFY COLUMN info.`` COMMENT 'comment'", + "ALTER TABLE t DROP COLUMN info.``", + "ALTER TABLE t RENAME COLUMN info.`` TO renamed", + "ALTER TABLE t ORDER BY (id, ``)")) { + ParseException exception = Assertions.assertThrows(ParseException.class, + () -> parser.parseSingle(sql), sql); + Assertions.assertTrue(exception.getMessage().contains("Quoted identifier cannot be empty"), sql); + } + } + + @Test + public void testEmptyQuotedIdentifiersOutsideColumnPathsKeepExistingParserSemantics() { + Assertions.assertDoesNotThrow(() -> parser.parseSingle("ALTER TABLE t CREATE BRANCH ``")); + Assertions.assertDoesNotThrow( + () -> parser.parseSingle("GRANT SELECT_PRIV ON `internal`.``.`` TO 'user1'")); + } + + @Test + public void testModifyColumnRoundTripPreservesNullabilityIntent() { + ModifyColumnOp omitted = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.metric BIGINT", + ModifyColumnOp.class, "info.metric"); + Assertions.assertFalse(omitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertFalse(omitted.toSql().contains(" BIGINT NULL ")); + + ModifyColumnOp reparsedOmitted = assertSingleClausePath( + "ALTER TABLE t " + omitted.toSql(), ModifyColumnOp.class, "info.metric"); + Assertions.assertFalse(reparsedOmitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + + ModifyColumnOp nullable = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.metric BIGINT NULL", + ModifyColumnOp.class, "info.metric"); + ModifyColumnOp reparsedNullable = assertSingleClausePath( + "ALTER TABLE t " + nullable.toSql(), ModifyColumnOp.class, "info.metric"); + Assertions.assertTrue(reparsedNullable.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + } + + @Test + public void testModifyColumnRoundTripPreservesCommentIntent() { + ModifyColumnOp omitted = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN arr.element BIGINT", + ModifyColumnOp.class, "arr.element"); + Assertions.assertFalse(omitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + Assertions.assertFalse(omitted.toSql().contains(" COMMENT ")); + + ModifyColumnOp reparsedOmitted = assertSingleClausePath( + "ALTER TABLE t " + omitted.toSql(), ModifyColumnOp.class, "arr.element"); + Assertions.assertFalse(reparsedOmitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + + ModifyColumnOp empty = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN arr.element BIGINT COMMENT ''", + ModifyColumnOp.class, "arr.element"); + ModifyColumnOp reparsedEmpty = assertSingleClausePath( + "ALTER TABLE t " + empty.toSql(), ModifyColumnOp.class, "arr.element"); + Assertions.assertTrue(reparsedEmpty.getColumnDef() + .translateToCatalogStyleForSchemaChange().isCommentSpecified()); + Assertions.assertEquals("", reparsedEmpty.getColumnDef().getComment()); + } + + @Test + public void testStructMemberRoundTripPreservesCommentIntent() { + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.payload " + + "STRUCT", + ModifyColumnOp.class, "info.payload"); + StructType structType = (StructType) modify.getColumnDef().getType(); + Assertions.assertFalse(structType.getFields().get(0).isCommentSpecified()); + Assertions.assertTrue(structType.getFields().get(1).isCommentSpecified()); + + ModifyColumnOp reparsed = assertSingleClausePath( + "ALTER TABLE t " + modify.toSql(), ModifyColumnOp.class, "info.payload"); + StructType reparsedType = (StructType) reparsed.getColumnDef().getType(); + Assertions.assertFalse(reparsedType.getFields().get(0).isCommentSpecified()); + Assertions.assertTrue(reparsedType.getFields().get(1).isCommentSpecified()); + Assertions.assertEquals("", reparsedType.getFields().get(1).getComment()); + org.apache.doris.catalog.StructType catalogType = (org.apache.doris.catalog.StructType) reparsed + .getColumnDef().translateToCatalogStyleForSchemaChange().getType(); + Assertions.assertFalse(catalogType.getFields().get(0).isCommentSpecified()); + Assertions.assertTrue(catalogType.getFields().get(1).isCommentSpecified()); + } + + @Test + public void testModifyColumnCommentRoundTripEscapesQuotesAndBackslashes() { + assertCommentRoundTrip(false); + assertCommentRoundTrip(true); + } + + @Test + public void testColumnDefinitionCommentRoundTripEscapesQuotesAndBackslashes() { + assertColumnDefinitionCommentRoundTrip(false); + assertColumnDefinitionCommentRoundTrip(true); + } + + @Test + public void testRegularColumnDefinitionCommentRoundTripEscapesQuotesAndBackslashes() { + assertRegularColumnDefinitionCommentRoundTrip(false); + assertRegularColumnDefinitionCommentRoundTrip(true); + } + + @Test + public void testStructMemberCommentRoundTripEscapesQuotesAndBackslashes() { + assertStructMemberCommentRoundTrip(false); + assertStructMemberCommentRoundTrip(true); + } + + private void assertStructMemberCommentRoundTrip(boolean noBackslashEscapes) { + try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); + + String sqlPath = noBackslashEscapes ? "C:\\tmp\\" : "C:\\\\tmp\\\\"; + String expectedComment = "owner's \"field\" C:\\tmp\\"; + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.payload " + + "STRUCT", + ModifyColumnOp.class, "info.payload"); + StructType structType = (StructType) modify.getColumnDef().getType(); + Assertions.assertEquals(expectedComment, structType.getFields().get(0).getComment()); + + ModifyColumnOp reparsedModify = assertSingleClausePath( + "ALTER TABLE t " + modify.toSql(), ModifyColumnOp.class, "info.payload"); + StructType reparsedType = (StructType) reparsedModify.getColumnDef().getType(); + Assertions.assertEquals(expectedComment, reparsedType.getFields().get(0).getComment()); + } + } + + private void assertStructMemberNames(StructType structType) { + Assertions.assertEquals("key", structType.getFields().get(0).getName()); + Assertions.assertEquals("metric`name", structType.getFields().get(1).getName()); + } + + private void assertRegularColumnDefinitionCommentRoundTrip(boolean noBackslashEscapes) { + try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); + + String sqlPath = noBackslashEscapes ? "C:\\tmp\\" : "C:\\\\tmp\\\\"; + String expectedComment = "owner's \"field\" C:\\tmp\\"; + AddColumnsOp addColumns = assertSingleClause( + "ALTER TABLE t ADD COLUMN (owner STRING NULL COMMENT 'owner''s \"field\" " + + sqlPath + "', metric INT NULL)", AddColumnsOp.class); + Assertions.assertEquals(expectedComment, + addColumns.getColumnDefinitions().get(0).getComment()); + + AddColumnsOp reparsedAddColumns = assertSingleClause( + "ALTER TABLE t " + addColumns.toSql(), AddColumnsOp.class); + Assertions.assertEquals(expectedComment, + reparsedAddColumns.getColumnDefinitions().get(0).getComment()); + } + } + + private void assertColumnDefinitionCommentRoundTrip(boolean noBackslashEscapes) { + try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); + + String sqlPath = noBackslashEscapes ? "C:\\tmp\\" : "C:\\\\tmp\\\\"; + String expectedComment = "owner's \"field\" C:\\tmp\\"; + AddColumnOp add = assertSingleClausePath( + "ALTER TABLE t ADD COLUMN info.owner STRING NULL COMMENT 'owner''s \"field\" " + + sqlPath + "'", + AddColumnOp.class, "info.owner"); + Assertions.assertEquals(expectedComment, add.getColumnDef().getComment()); + AddColumnOp reparsedAdd = assertSingleClausePath( + "ALTER TABLE t " + add.toSql(), AddColumnOp.class, "info.owner"); + Assertions.assertEquals(expectedComment, reparsedAdd.getColumnDef().getComment()); + + ModifyColumnOp modify = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.owner STRING COMMENT 'owner''s \"field\" " + + sqlPath + "'", + ModifyColumnOp.class, "info.owner"); + Assertions.assertEquals(expectedComment, modify.getColumnDef().getComment()); + ModifyColumnOp reparsedModify = assertSingleClausePath( + "ALTER TABLE t " + modify.toSql(), ModifyColumnOp.class, "info.owner"); + Assertions.assertEquals(expectedComment, reparsedModify.getColumnDef().getComment()); + } + } + + private void assertCommentRoundTrip(boolean noBackslashEscapes) { + try (MockedStatic mockedSqlMode = Mockito.mockStatic(SqlModeHelper.class)) { + mockedSqlMode.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(noBackslashEscapes); + + String expectedComment = "owner's \"field\" C:\\tmp\\"; + ModifyColumnCommentOp comment = new ModifyColumnCommentOp( + ColumnPath.fromDotName("info.metric"), expectedComment); + String renderedSql = comment.toSql(); + Assertions.assertTrue(renderedSql.contains("\"\"field\"\"")); + Assertions.assertTrue(renderedSql.contains(noBackslashEscapes + ? "C:\\tmp\\" : "C:\\\\tmp\\\\")); + + ModifyColumnCommentOp reparsed = assertSingleClausePath( + "ALTER TABLE t " + renderedSql, ModifyColumnCommentOp.class, "info.metric"); + Assertions.assertEquals(expectedComment, reparsed.getComment()); + + ModifyColumnCommentOp doubledSingleQuote = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.metric COMMENT 'owner''s'", + ModifyColumnCommentOp.class, "info.metric"); + Assertions.assertEquals("owner's", doubledSingleQuote.getComment()); + ModifyColumnCommentOp doubledDoubleQuote = assertSingleClausePath( + "ALTER TABLE t MODIFY COLUMN info.metric COMMENT \"a\"\"b\"", + ModifyColumnCommentOp.class, "info.metric"); + Assertions.assertEquals("a\"b", doubledDoubleQuote.getComment()); + } + } + + private T assertSingleClausePath(String sql, Class clauseClass, + String expectedPath) { + T clause = assertSingleClause(sql, clauseClass); + if (clause instanceof AddColumnOp) { + Assertions.assertEquals(expectedPath, ((AddColumnOp) clause).getColumnPath().getFullPath()); + } else if (clause instanceof ModifyColumnOp) { + Assertions.assertEquals(expectedPath, ((ModifyColumnOp) clause).getColumnPath().getFullPath()); + } else if (clause instanceof ModifyColumnCommentOp) { + Assertions.assertEquals(expectedPath, ((ModifyColumnCommentOp) clause).getColumnPath().getFullPath()); + } else if (clause instanceof DropColumnOp) { + Assertions.assertEquals(expectedPath, ((DropColumnOp) clause).getColumnPath().getFullPath()); + } else if (clause instanceof RenameColumnOp) { + Assertions.assertEquals(expectedPath, ((RenameColumnOp) clause).getColumnPath().getFullPath()); + } + return clause; + } + + private T assertSingleClause(String sql, Class clauseClass) { + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(AlterTableCommand.class, plan); + List clauses = ((AlterTableCommand) plan).getOps(); + Assertions.assertEquals(1, clauses.size()); + AlterTableOp clause = clauses.get(0); + Assertions.assertInstanceOf(clauseClass, clause); + return clauseClass.cast(clause); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java index cb5045ad76dd8c..ee0942c30716d9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopnLazyMaterializeTest.java @@ -24,6 +24,8 @@ import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.processor.post.PlanPostProcessors; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.MaterializationNode; @@ -117,6 +119,29 @@ public void testNestedColumnAccessPathInLazyMaterialize() throws Exception { ColumnAccessPath.data(ImmutableList.of("user_profile", "professional", "skills")))); } + @Test + public void testLazyBaseColumnIndexUsesOriginalColumnNameForAlias() throws Exception { + this.createTable("create table lazy_materialize_alias_tbl(" + + "sort_col int, lazy_col int) " + + "duplicate key(sort_col) distributed by hash(sort_col) buckets 1 " + + "properties('replication_num' = '1')"); + String sql = "select lazy_col as lazy_alias from lazy_materialize_alias_tbl " + + "order by sort_col limit 1"; + + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .implement(); + PhysicalPlan plan = checker.getPhysicalPlan(); + plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan); + + List> materializeNodes = plan.collectToList( + node -> node instanceof PhysicalLazyMaterialize); + Assertions.assertEquals(1, materializeNodes.size(), plan.treeString()); + Assertions.assertEquals(ImmutableList.of(ImmutableList.of(1)), + materializeNodes.get(0).getLazyBaseColumnIndices()); + } + @Test public void testLightSchemaChangeFalse() throws Exception { this.createTable("create table tm_lsc_false (k int, v int) duplicate key(k) " diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregateTest.java index 52d7068db080c4..37aa9d4913a33a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregateTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -736,6 +737,15 @@ public void testAggregateOrderByExpressionNeedPushDown() { ); } + @Test + public void testAggregateOrderByExpressionCannotContainAggregateFunction() { + String sql = "select group_concat(k order by sum(k)) as s " + + "from (select 1 as k union all select 2) t"; + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext).analyze(sql)); + Assertions.assertEquals("aggregate function cannot contain aggregate parameters", exception.getMessage()); + } + @Test public void testDistinctAggregateOrderByExpressionNeedPushDown() { String sql = "select group_concat(distinct name order by id + no) from t1"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeMultiColumnDistinctCountTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeMultiColumnDistinctCountTest.java new file mode 100644 index 00000000000000..70bb1699949c86 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeMultiColumnDistinctCountTest.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.AggregateUtils; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.nereids.util.PlanConstructor; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class NormalizeMultiColumnDistinctCountTest implements MemoPatternMatchSupported { + @Test + public void testNormalizeDifferentArgumentOrder() { + Plan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, + ImmutableList.of()); + Slot id = student.getOutput().get(0); + Slot name = student.getOutput().get(2); + Alias countIdName = new Alias(new Count(true, id, name), "count_id_name"); + Alias countNameId = new Alias(new Count(true, name, id), "count_name_id"); + LogicalAggregate root = new LogicalAggregate<>(ImmutableList.of(), + ImmutableList.of(countIdName, countNameId), student); + + PlanChecker.from(MemoTestUtils.createConnectContext(), root) + .applyTopDown(new NormalizeAggregate()) + .matchesFromRoot( + logicalProject( + logicalAggregate( + logicalProject( + logicalOlapScan() + ) + ).when(aggregate -> aggregate.getAggregateFunctions().size() == 1) + .when(aggregate -> aggregate.getOutputExpressions().size() == 1) + ).when(project -> project.getProjects().size() == 2) + .when(project -> project.getProjects().get(0).getExprId() + .equals(countIdName.getExprId())) + .when(project -> project.getProjects().get(1).getExprId() + .equals(countNameId.getExprId())) + .when(project -> project.getProjects().get(0).getInputSlots() + .equals(project.getProjects().get(1).getInputSlots())) + ); + } + + @Test + public void testKeepDifferentDistinctArguments() { + Plan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, + ImmutableList.of()); + Slot id = student.getOutput().get(0); + Slot gender = student.getOutput().get(1); + Slot name = student.getOutput().get(2); + LogicalAggregate root = new LogicalAggregate<>(ImmutableList.of(), + ImmutableList.of( + new Alias(new Count(true, id, name), "count_id_name"), + new Alias(new Count(true, id, gender), "count_id_gender")), + student); + + PlanChecker.from(MemoTestUtils.createConnectContext(), root) + .applyTopDown(new NormalizeAggregate()) + .matches( + logicalJoin( + logicalAggregate() + .when(aggregate -> aggregate.getAggregateFunctions().size() == 1), + logicalAggregate() + .when(aggregate -> aggregate.getAggregateFunctions().size() == 1) + ) + ); + } + + @Test + public void testRemoveDuplicateDistinctArguments() { + Plan student = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, + ImmutableList.of()); + Slot id = student.getOutput().get(0); + Slot name = student.getOutput().get(2); + Count count = new Count(true, id, id, name); + + Count countIf = (Count) AggregateUtils.countDistinctMultiExprToCountIf(count); + Assertions.assertEquals(ImmutableSet.of(id, name), countIf.getInputSlots()); + + LogicalAggregate root = new LogicalAggregate<>(ImmutableList.of(), + ImmutableList.of( + new Alias(count, "count_id_id_name"), + new Alias(new Count(true, name, id), "count_name_id")), + student); + PlanChecker.from(MemoTestUtils.createConnectContext(), root) + .applyTopDown(new NormalizeAggregate()) + .matchesFromRoot( + logicalProject( + logicalAggregate( + logicalProject( + logicalOlapScan() + ) + ).when(aggregate -> aggregate.getAggregateFunctions().size() == 1) + .when(aggregate -> aggregate.getAggregateFunctions().iterator().next() + .arity() == 2) + ) + ); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BinarySearchPartitionInconsistencyTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BinarySearchPartitionInconsistencyTest.java new file mode 100644 index 00000000000000..db3d46489f38ff --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BinarySearchPartitionInconsistencyTest.java @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionPruneResult; +import org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Optional; + +/** + * Verify fix for the TOCTOU bug in {@link SelectedPartitions}. + * + *

Before the fix: {@code nameToPartitionItem} was frozen in {@code SelectedPartitions} + * at T1 but {@code sortedPartitionRanges} was re-read from cache at T2. If the cache + * changed between T1 and T2 (concurrent ADD/DROP PARTITION), the two snapshots diverged, + * binary search returned partitions not present in {@code nameToPartitionItem}, and + * {@code ImmutableMap.copyOf()} threw NPE on the null value. + * + *

After the fix: {@code sortedPartitionRanges} is built from the same + * {@code nameToPartitionItems} and frozen inside {@code SelectedPartitions} at T1. + * The pruning rule reads both from the frozen snapshot so they are always consistent. + */ +public class BinarySearchPartitionInconsistencyTest extends TestWithFeService { + private final Column partitionColumn = new Column("a", PrimitiveType.INT); + private final SlotReference slotA = new SlotReference("a", IntegerType.INSTANCE); + private CascadesContext cascadesContext; + + @Override + protected void runBeforeAll() throws Exception { + cascadesContext = createCascadesContext("select * from t1"); + } + + private ListPartitionItem listItem(int value) throws AnalysisException { + PartitionValue partitionValue = new PartitionValue(String.valueOf(value)); + PartitionKey partitionKey = PartitionKey.createPartitionKey( + ImmutableList.of(partitionValue), ImmutableList.of(partitionColumn)); + return new ListPartitionItem(ImmutableList.of(partitionKey)); + } + + /** + * Test that SelectedPartitions correctly freezes sortedPartitionRanges + * from the same partition items map. + */ + @Test + public void testSelectedPartitionsFreezesSortedPartitionRanges() throws AnalysisException { + Map nameToPartitionItems = Maps.newHashMapWithExpectedSize(3); + nameToPartitionItems.put("p1", listItem(1)); + nameToPartitionItems.put("p2", listItem(2)); + nameToPartitionItems.put("p3", listItem(3)); + + Optional> sortedRanges = Optional.ofNullable( + SortedPartitionRanges.build(nameToPartitionItems)); + + SelectedPartitions sp = new SelectedPartitions( + nameToPartitionItems.size(), nameToPartitionItems, false, false, sortedRanges); + + // sortedPartitionRanges field is populated + Assertions.assertTrue(sp.sortedPartitionRanges.isPresent(), + "sortedPartitionRanges should be present when partitions exist"); + Assertions.assertNotNull(sp.sortedPartitionRanges.get(), + "sortedPartitionRanges should not be null"); + } + + /** + * Test that NOT_PRUNED has empty sortedPartitionRanges. + */ + @Test + public void testNotPrunedHasEmptySortedPartitionRanges() { + Assertions.assertFalse(SelectedPartitions.NOT_PRUNED.sortedPartitionRanges.isPresent(), + "NOT_PRUNED should have empty sortedPartitionRanges"); + Assertions.assertEquals(Optional.empty(), SelectedPartitions.NOT_PRUNED.sortedPartitionRanges, + "NOT_PRUNED.sortedPartitionRanges should be Optional.empty()"); + } + + /** + * Test consistent snapshot: when both nameToPartitionItem and sortedPartitionRanges + * come from the same data, no partition returned by binary search is missing + * from the map. Simulates the PruneFileScanPartition flow after the fix. + */ + @Test + public void testConsistentSnapshotNoMissingPartitions() throws AnalysisException { + // One consistent snapshot built at T1 + Map partitionItems = Maps.newHashMapWithExpectedSize(4); + partitionItems.put("p1", listItem(1)); + partitionItems.put("p2", listItem(2)); + partitionItems.put("p3", listItem(3)); + partitionItems.put("p4", listItem(4)); + partitionItems = ImmutableMap.copyOf(partitionItems); + + SortedPartitionRanges sortedRanges = SortedPartitionRanges.build(partitionItems); + Assertions.assertNotNull(sortedRanges); + + // predicate hits p4, which DOES exist in both snapshots + Expression predicate = new EqualTo(slotA, Literal.of(4)); + + PartitionPruneResult result = PartitionPruner.pruneWithResult( + ImmutableList.of(slotA), predicate, partitionItems, cascadesContext, + PartitionTableType.EXTERNAL, Optional.of(sortedRanges)); + + // p4 should be in the result since it exists in the map + Assertions.assertTrue(result.partitions.contains("p4"), + "p4 should be returned when it exists in the consistent snapshot"); + + // Simulate the PruneFileScanPartition lookup loop. Since both the partition map + // and sortedPartitionRanges come from the same frozen snapshot, every returned + // partition must be present in the map — assert the invariant rather than skip. + Map selectedPartitionItems = Maps.newHashMap(); + for (String name : result.partitions) { + PartitionItem item = partitionItems.get(name); + Assertions.assertNotNull(item, + "pruned partition " + name + " must be present in the consistent snapshot"); + selectedPartitionItems.put(name, item); + } + + // All returned partitions are in the map, none skipped + Assertions.assertEquals(result.partitions.size(), selectedPartitionItems.size(), + "all returned partitions should be present in the consistent snapshot"); + Assertions.assertNotNull(selectedPartitionItems.get("p4"), + "p4 PartitionItem should not be null"); + } + + /** + * Test: if inconsistent snapshots ever reach the lookup loop (which the fix makes + * impossible by freezing both from the same snapshot), the invariant check fails + * loudly instead of silently returning a partial scan over fewer partitions. + * + *

This mirrors the {@code Preconditions.checkState} in PruneFileScanPartition: + * a missing partition is treated as an invariant violation, not a partition to skip. + */ + @Test + public void testInvariantViolationFailsLoudly() throws AnalysisException { + // old snapshot (no p4) + Map nameToPartitionItem = ImmutableMap.of( + "p1", listItem(1), + "p2", listItem(2), + "p3", listItem(3)); + + // new snapshot (has p4) — simulating inconsistent data that the fix prevents, + // but the invariant check must still surface it rather than silently skip p4 + Map newPartitions = Maps.newHashMapWithExpectedSize(4); + newPartitions.put("p1", listItem(1)); + newPartitions.put("p2", listItem(2)); + newPartitions.put("p3", listItem(3)); + newPartitions.put("p4", listItem(4)); + SortedPartitionRanges sortedPartitionRanges = SortedPartitionRanges.build(newPartitions); + Assertions.assertNotNull(sortedPartitionRanges); + + Expression predicate = new EqualTo(slotA, Literal.of(4)); + + PartitionPruneResult result = PartitionPruner.pruneWithResult( + ImmutableList.of(slotA), predicate, nameToPartitionItem, cascadesContext, + PartitionTableType.EXTERNAL, Optional.of(sortedPartitionRanges)); + + // binary search returns p4 from the new snapshot + Assertions.assertTrue(result.partitions.contains("p4"), + "binary search returns p4 from the sortedPartitionRanges"); + + // The invariant check (mirroring PruneFileScanPartition) must fail loudly: + // p4 is in the pruned result but missing from nameToPartitionItem. + Assertions.assertThrows(IllegalStateException.class, () -> { + for (String name : result.partitions) { + PartitionItem item = nameToPartitionItem.get(name); + if (item == null) { + throw new IllegalStateException( + "pruned partition " + name + " is missing in the selected partitions snapshot"); + } + } + }, "a missing partition must fail the invariant instead of being silently skipped"); + } + + /** + * Test that with binary search disabled, sequentialFiltering is unaffected. + */ + @Test + public void testSequentialFilteringNotAffectedByInconsistentSnapshot() throws AnalysisException { + Map nameToPartitionItem = Maps.newHashMapWithExpectedSize(3); + nameToPartitionItem.put("p1", listItem(1)); + nameToPartitionItem.put("p2", listItem(2)); + nameToPartitionItem.put("p3", listItem(3)); + nameToPartitionItem = ImmutableMap.copyOf(nameToPartitionItem); + + Expression predicate = new EqualTo(slotA, Literal.of(4)); + + // binary search disabled -> sequentialFiltering path + PartitionPruneResult result = PartitionPruner.pruneWithResult( + ImmutableList.of(slotA), predicate, nameToPartitionItem, cascadesContext, + PartitionTableType.EXTERNAL, Optional.empty()); + + Assertions.assertFalse(result.partitions.contains("p4"), + "sequential filtering only uses nameToPartitionItem, p4 is never returned"); + Assertions.assertTrue(result.partitions.isEmpty(), + "no partition matches a=4 in this snapshot"); + } + + /** + * Test that SelectedPartitions factory method produces consistent state. + */ + @Test + public void testNewSelectedPartitionsHasConsistentSortedRanges() throws AnalysisException { + Map items = Maps.newHashMapWithExpectedSize(2); + items.put("p1", listItem(1)); + items.put("p2", listItem(2)); + + Optional> ranges = Optional.ofNullable( + SortedPartitionRanges.build(items)); + + // The 5-arg constructor freezes both fields from the same snapshot + SelectedPartitions sp = new SelectedPartitions(items.size(), items, false, false, ranges); + + Assertions.assertTrue(sp.sortedPartitionRanges.isPresent()); + SortedPartitionRanges frozen = sp.sortedPartitionRanges.get(); + + // The frozen sortedPartitionRanges contains exactly the same partitions as selectedPartitions + int partitionCountInSortedRanges = frozen.sortedPartitions.size() + frozen.defaultPartitions.size(); + Assertions.assertEquals(sp.selectedPartitions.size(), partitionCountInSortedRanges, + "sortedPartitionRanges should cover all partitions in selectedPartitions"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java index bf5b74f2ea6f25..d492100e867f29 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PhysicalStorageLayerAggregateTest.java @@ -17,6 +17,12 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; @@ -27,7 +33,10 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.scalar.Ln; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate.PushDownAggOp; @@ -38,6 +47,7 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.Collections; import java.util.Optional; @@ -75,7 +85,26 @@ public void testWithoutProject() { .applyImplementation(storageLayerAggregateWithoutProject()) .matches( logicalAggregate( - physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT) + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().equals( + ImmutableList.of(olapScan.getOutput().get(0).getExprId()))) + ) + ); + + // COUNT(*) still keeps a placeholder scan slot after column pruning, so its semantic + // argument list must remain empty instead of being inferred from the physical scan shape. + aggregate = new LogicalAggregate<>( + Collections.emptyList(), + ImmutableList.of(new Alias(new Count(), "count_star")), + true, Optional.empty(), olapScan); + context = MemoTestUtils.createCascadesContext(aggregate); + + PlanChecker.from(context) + .applyImplementation(storageLayerAggregateWithoutProject()) + .matches( + logicalAggregate( + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().isEmpty()) ) ); @@ -96,6 +125,67 @@ public void testWithoutProject() { ); } + @Test + public void testNullableFileCountUsesStorageLayerAggregate() { + LogicalAggregate aggregate = newNullableFileCountAggregate(); + LogicalFileScan fileScan = aggregate.child(); + + PlanChecker.from(MemoTestUtils.createCascadesContext(aggregate)) + .applyImplementation(storageLayerAggregateWithoutProjectForFileScan()) + .matches(logicalAggregate( + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().equals( + ImmutableList.of(fileScan.getOutput().get(0).getExprId()))))); + } + + @Test + public void testNullableFileCountDoesNotUseV1StorageLayerAggregate() { + LogicalAggregate aggregate = newNullableFileCountAggregate(); + CascadesContext context = MemoTestUtils.createCascadesContext(aggregate); + context.getConnectContext().getSessionVariable().enableFileScannerV2 = false; + + PlanChecker.from(context) + .applyImplementation(storageLayerAggregateWithoutProjectForFileScan()) + .nonMatch(physicalStorageLayerAggregate()); + } + + @Test + public void testMixedCountStarAndNullableFileCountDoesNotUseStorageLayerAggregate() { + LogicalAggregate nullableCount = newNullableFileCountAggregate(); + LogicalFileScan fileScan = nullableCount.child(); + LogicalAggregate mixedCount = new LogicalAggregate<>( + Collections.emptyList(), + ImmutableList.of(new Alias(new Count(), "count_star"), + new Alias(new Count(fileScan.getOutput().get(0)), "count_nullable")), + true, Optional.empty(), fileScan); + + PlanChecker.from(MemoTestUtils.createCascadesContext(mixedCount)) + .applyImplementation(storageLayerAggregateWithoutProjectForFileScan()) + .nonMatch(physicalStorageLayerAggregate()); + } + + private LogicalAggregate newNullableFileCountAggregate() { + Column nullableColumn = new Column("value", Type.INT, true); + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.initSelectedPartitions(Mockito.any())) + .thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(table.getFullSchema()).thenReturn(ImmutableList.of(nullableColumn)); + Mockito.when(table.getName()).thenReturn("nullable_file_table"); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(catalog.getName()).thenReturn("catalog"); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(table.getDatabase()).thenReturn(database); + LogicalFileScan fileScan = new LogicalFileScan(new RelationId(1), table, + ImmutableList.of("catalog", "db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + return new LogicalAggregate<>( + Collections.emptyList(), + ImmutableList.of(new Alias(new Count(fileScan.getOutput().get(0)), "count")), + true, Optional.empty(), fileScan); + } + @Override public RulePromise defaultPromise() { return RulePromise.IMPLEMENT; @@ -138,7 +228,9 @@ public void testWithProject() { .matches( logicalAggregate( logicalProject( - physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT) + physicalStorageLayerAggregate().when(agg -> agg.getAggOp() == PushDownAggOp.COUNT + && agg.getCountArgumentExprIds().equals( + ImmutableList.of(olapScan.getOutput().get(0).getExprId()))) ) ) ); @@ -197,6 +289,15 @@ private Rule storageLayerAggregateWithoutProject() { .get(); } + private Rule storageLayerAggregateWithoutProjectForFileScan() { + return new AggregateStrategies().buildRules() + .stream() + .filter(rule -> rule.getRuleType() + == RuleType.STORAGE_LAYER_AGGREGATE_WITHOUT_PROJECT_FOR_FILE_SCAN) + .findFirst() + .get(); + } + private Rule storageLayerAggregateWithProject() { return new AggregateStrategies().buildRules() .stream() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index efbe72a75acecd..3c92606e4e3fcc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -1022,15 +1022,24 @@ public void testDataTypeAccessTree() { Assertions.assertEquals("struct>>>", columnType.toSql()); setAccessPathAndAssertType(slot, ImmutableList.of("s", "city"), "STRUCT"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "KEYS"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "a"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "b"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "a"), "STRUCT>>>"); - setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "b"), "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data"), + "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*"), + "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "KEYS"), + "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES"), + "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "a"), + "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "VALUES", "b"), + "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*"), + "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "a"), + "STRUCT>>>"); + setAccessPathAndAssertType(slot, ImmutableList.of("s", "data", "*", "*", "b"), + "STRUCT>>>"); setAccessPathsAndAssertType(slot, ImmutableList.of( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionShapeInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionShapeInfoTest.java new file mode 100644 index 00000000000000..72cc8bd0ed0619 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionShapeInfoTest.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam; +import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.types.DoubleType; +import org.apache.doris.nereids.types.IntegerType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class ExpressionShapeInfoTest { + + @Test + void testWrappedAggregateShapeInfoPreservesQualifier() { + assertPreservesQualifier(new Cast(new Abs(avg()), DoubleType.INSTANCE)); + assertPreservesQualifier(new TryCast(avg(), DoubleType.INSTANCE)); + assertPreservesQualifier(new BitNot(avg())); + assertPreservesQualifier(new Alias(new Cast(avg(), DoubleType.INSTANCE), "avg_alias")); + assertPreservesQualifier(new AggregateExpression(avg(), AggregateParam.LOCAL_RESULT)); + assertPreservesQualifier(new Not(new IsNull(avg()))); + assertPreservesQualifier(new Between(avg(), new IntegerLiteral(1), new IntegerLiteral(2))); + assertPreservesQualifier(new InPredicate(avg(), ImmutableList.of(new IntegerLiteral(1)))); + assertPreservesQualifier(new CaseWhen( + ImmutableList.of(new WhenClause(new IsNull(avg()), new Cast(avg(), DoubleType.INSTANCE))), + new IntegerLiteral(0))); + } + + private void assertPreservesQualifier(Expression expression) { + String shapeInfo = expression.shapeInfo(); + Assertions.assertTrue(shapeInfo.contains("avg(t1.c11)"), shapeInfo); + Assertions.assertFalse(shapeInfo.contains("avg(c11)"), shapeInfo); + } + + private Avg avg() { + return new Avg(new SlotReference("c11", IntegerType.INSTANCE, true, ImmutableList.of("t1"))); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java new file mode 100644 index 00000000000000..0bf26dd0da86d5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.executable; + +import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; +import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class StringArithmeticTest { + + @Test + void testFieldMatchesFloatNaN() { + IntegerLiteral result = (IntegerLiteral) StringArithmetic.fieldFloat(new FloatLiteral(Float.NaN), + new FloatLiteral(Float.NaN)); + + Assertions.assertEquals(1, result.getValue()); + } + + @Test + void testFieldMatchesDoubleNaN() { + IntegerLiteral result = (IntegerLiteral) StringArithmetic.fieldDouble(new DoubleLiteral(Double.NaN), + new DoubleLiteral(Double.NaN)); + + Assertions.assertEquals(1, result.getValue()); + } + + @Test + void testFieldMatchesFloatSignedZero() { + IntegerLiteral positiveZero = (IntegerLiteral) StringArithmetic.fieldFloat(new FloatLiteral(0.0f), + new FloatLiteral(-0.0f)); + IntegerLiteral negativeZero = (IntegerLiteral) StringArithmetic.fieldFloat(new FloatLiteral(-0.0f), + new FloatLiteral(0.0f)); + + Assertions.assertEquals(1, positiveZero.getValue()); + Assertions.assertEquals(1, negativeZero.getValue()); + } + + @Test + void testFieldMatchesDoubleSignedZero() { + IntegerLiteral positiveZero = (IntegerLiteral) StringArithmetic.fieldDouble(new DoubleLiteral(0.0), + new DoubleLiteral(-0.0)); + IntegerLiteral negativeZero = (IntegerLiteral) StringArithmetic.fieldDouble(new DoubleLiteral(-0.0), + new DoubleLiteral(0.0)); + + Assertions.assertEquals(1, positiveZero.getValue()); + Assertions.assertEquals(1, negativeZero.getValue()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java new file mode 100644 index 00000000000000..abb37cfc9198a6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.DateV2Type; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class StrToDateTest { + + @Test + void testComputeSignatureWithFoldableFormat() { + StrToDate dateFormat = new StrToDate(new StringLiteral("2024-01-01"), + new Concat(new StringLiteral("%Y-%m"), new StringLiteral("-%d"))); + StrToDate dateTimeFormat = new StrToDate(new StringLiteral("2024-01-01 12:34:56"), + new Concat(new StringLiteral("%Y-%m-%d"), new StringLiteral(" %H:%i:%s"))); + + FunctionSignature dateSignature = dateFormat.computeSignature(StrToDate.SIGNATURES.get(0)); + FunctionSignature dateTimeSignature = dateTimeFormat.computeSignature(StrToDate.SIGNATURES.get(0)); + + Assertions.assertEquals(DateV2Type.INSTANCE, dateSignature.returnType); + Assertions.assertEquals(DateTimeV2Type.SYSTEM_DEFAULT, dateTimeSignature.returnType); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java index f3bb60d0d36c11..8b2db5b3511e8b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java @@ -40,10 +40,9 @@ class DateLiteralTest { @Test void reject() { - Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 01:00:00.000000")); - Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:01:00.000000")); - Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:00:01.000000")); - Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:00:00.000001")); + Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 24:00:00.000000")); + Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:60:00.000000")); + Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:00:60.000000")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DecimalLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DecimalLiteralTest.java index 7e5cf346b0e28c..cdb087c7ff1595 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DecimalLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DecimalLiteralTest.java @@ -130,4 +130,14 @@ void testUncheckedCastTo() { Assertions.assertInstanceOf(StringLiteral.class, expression); Assertions.assertEquals("234.567", ((StringLiteral) expression).value); } + + @Test + void testDecimalV3CastToStringUsesPlainNotation() { + DecimalV3Literal literal = new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(10, 2), + new BigDecimal("1E+3")); + + StringLiteral string = (StringLiteral) literal.uncheckedCastTo(StringType.INSTANCE); + + Assertions.assertEquals("1000.00", string.getStringValue()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java index b1b58520b200aa..0675a07c60708e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java @@ -20,6 +20,8 @@ import org.apache.doris.nereids.exceptions.CastException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.DateType; +import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.FloatType; @@ -204,6 +206,17 @@ void testUncheckedCastTo() { } + @Test + void testCastMaxDateTimeStringToDateDoesNotRoundTimePart() { + StringLiteral literal = new StringLiteral("9999-12-31 23:59:59.999999"); + + DateLiteral date = (DateLiteral) literal.uncheckedCastTo(DateType.INSTANCE); + DateV2Literal dateV2 = (DateV2Literal) literal.uncheckedCastTo(DateV2Type.INSTANCE); + + Assertions.assertEquals("9999-12-31", date.getStringValue()); + Assertions.assertEquals("9999-12-31", dateV2.getStringValue()); + } + @Test void testUncheckedCastToTimeStampTzRejectsUnstrictNoOffsetInStrictMode() { try (MockedStatic mockedSessionVariable = Mockito.mockStatic(SessionVariable.class)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java index ef73fe1efd4379..96775d44bfc628 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TimeV2Type; @@ -174,4 +175,15 @@ public void testUncheckedCast() { Assertions.assertEquals("00:00:00.000", ((StringLiteral) expression).value); } + @Test + public void testCastNegativeTimeToDateTimeV2KeepsSign() { + TimeV2Literal literal = new TimeV2Literal(TimeV2Type.of(0), "-00:00:01"); + + DateTimeV2Literal dateTime = (DateTimeV2Literal) literal.uncheckedCastTo(DateTimeV2Type.of(0)); + + Assertions.assertEquals(23, dateTime.getHour()); + Assertions.assertEquals(59, dateTime.getMinute()); + Assertions.assertEquals(59, dateTime.getSecond()); + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainTableStreamPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainTableStreamPlanTest.java index 9feaba171e5231..edc47a7dce5575 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainTableStreamPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainTableStreamPlanTest.java @@ -500,6 +500,24 @@ public void testAppendOnlyAliasedStreamResetCanBePlanned() throws Exception { assertStreamScanCanBePlanned(ctx, "explain select s.* from test_stream.s_dup@reset() as s"); } + @Test + public void testStreamScanWithSelectedPartitionIdsMarksPartitionPruned() { + Plan analyzedPlan = PlanChecker.from(connectContext) + .analyze("select * from test_stream.s2 where k1 < 100") + .getCascadesContext() + .getRewritePlan(); + + LogicalOlapTableStreamScan streamScan = findFirstLogicalStreamScan(analyzedPlan); + Assertions.assertNotNull(streamScan); + Assertions.assertFalse(streamScan.isPartitionPruned()); + + LogicalOlapTableStreamScan prunedScan = + streamScan.withSelectedPartitionIds(streamScan.getSelectedPartitionIds(), false); + + Assertions.assertTrue(prunedScan.isPartitionPruned()); + Assertions.assertFalse(prunedScan.hasPartitionPredicate()); + } + @Test public void testRecordPlanForMvPreRewriteNormalizesStreamScanInsideCte() throws Exception { ConnectContext ctx = createDefaultCtx(); @@ -577,6 +595,19 @@ private LogicalProject findFirstLogicalProject(Plan plan) { return null; } + private LogicalOlapTableStreamScan findFirstLogicalStreamScan(Plan plan) { + if (plan instanceof LogicalOlapTableStreamScan) { + return (LogicalOlapTableStreamScan) plan; + } + for (Plan child : plan.children()) { + LogicalOlapTableStreamScan found = findFirstLogicalStreamScan(child); + if (found != null) { + return found; + } + } + return null; + } + private boolean containsLogicalStreamScan(Plan plan) { if (plan instanceof LogicalOlapTableStreamScan) { return true; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterComputeGroupCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterComputeGroupCommandTest.java index ce11d4ce7f7723..dc5c595b77e88d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterComputeGroupCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterComputeGroupCommandTest.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.catalog.Env; -import org.apache.doris.cloud.catalog.ComputeGroup; +import org.apache.doris.cloud.catalog.CloudComputeGroupMeta; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -113,7 +113,7 @@ public void testValidateVirtualComputeGroup() throws Exception { Config.deploy_mode = "cloud"; CloudSystemInfoService mockCloudSIS = Mockito.mock(CloudSystemInfoService.class); - ComputeGroup mockComputeGroup = Mockito.mock(ComputeGroup.class); + CloudComputeGroupMeta mockComputeGroup = Mockito.mock(CloudComputeGroupMeta.class); Mockito.doReturn(mockComputeGroup).when(mockCloudSIS).getComputeGroupByName("virtual_group"); Mockito.doReturn(true).when(mockComputeGroup).isVirtual(); Deencapsulation.setField(env, "systemInfo", mockCloudSIS); @@ -130,7 +130,7 @@ public void testValidateInvalidProperties() throws Exception { Config.deploy_mode = "cloud"; CloudSystemInfoService mockCloudSIS = Mockito.mock(CloudSystemInfoService.class); - ComputeGroup mockComputeGroup = Mockito.mock(ComputeGroup.class); + CloudComputeGroupMeta mockComputeGroup = Mockito.mock(CloudComputeGroupMeta.class); Mockito.doReturn(mockComputeGroup).when(mockCloudSIS).getComputeGroupByName("test_group"); Mockito.doReturn(false).when(mockComputeGroup).isVirtual(); Mockito.doThrow(new DdlException("Invalid property")).when(mockComputeGroup) @@ -149,7 +149,7 @@ public void testValidateSuccess() throws Exception { Config.deploy_mode = "cloud"; CloudSystemInfoService mockCloudSIS = Mockito.mock(CloudSystemInfoService.class); - ComputeGroup mockComputeGroup = Mockito.mock(ComputeGroup.class); + CloudComputeGroupMeta mockComputeGroup = Mockito.mock(CloudComputeGroupMeta.class); Mockito.doReturn(mockComputeGroup).when(mockCloudSIS).getComputeGroupByName("test_group"); Mockito.doReturn(false).when(mockComputeGroup).isVirtual(); Mockito.doNothing().when(mockComputeGroup).checkProperties(Mockito.anyMap()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java index db64fe354d5cc3..f248e4382a0436 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommandTest.java @@ -17,22 +17,38 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.info.TableNameInfo; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.info.AddColumnsOp; import org.apache.doris.nereids.trees.plans.commands.info.AddPartitionFieldOp; import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp; import org.apache.doris.nereids.trees.plans.commands.info.DropPartitionFieldOp; import org.apache.doris.nereids.trees.plans.commands.info.EnableFeatureOp; +import org.apache.doris.nereids.trees.plans.commands.info.ModifyColumnOp; import org.apache.doris.nereids.trees.plans.commands.info.ReplacePartitionFieldOp; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; public class AlterTableCommandTest { + private final NereidsParser parser = new NereidsParser(); + @Test void testEnableFeatureOp() { List ops = new ArrayList<>(); @@ -155,4 +171,251 @@ void testReplacePartitionFieldOp() { "ALTER TABLE `internal`.`db`.`test` REPLACE PARTITION KEY bucket(16, id) WITH truncate(5, code) AS code_trunc", alterTableCommand.toSql()); } + + @Test + void testRejectNestedColumnPathForNonIcebergTable() { + TableIf table = Mockito.mock(TableIf.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN s.c STRING NULL", + "ALTER TABLE t MODIFY COLUMN s.a BIGINT", + "ALTER TABLE t MODIFY COLUMN s.a COMMENT 'nested comment'", + "ALTER TABLE t DROP COLUMN s.c", + "ALTER TABLE t RENAME COLUMN s.c TO c2")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Nested column path is only supported for Iceberg tables")); + } + } + + @Test + void testAllowNestedColumnPathForIcebergTable() throws AnalysisException { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + AlterTableCommand.checkColumnOperationsSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN s.c STRING NULL").getOps()); + } + + @Test + void testRejectRequiredNestedColumnForIcebergTable() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN s.required_field INT NOT NULL").getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("New nested field 's.required_field' must be nullable")); + } + + @Test + void testRejectRollupForIcebergColumnOperations() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN c STRING NULL TO r1", + "ALTER TABLE t ADD COLUMN s.c STRING NULL TO r1", + "ALTER TABLE t ADD COLUMN (c1 STRING NULL, c2 INT NULL) IN r1", + "ALTER TABLE t DROP COLUMN c FROM r1", + "ALTER TABLE t DROP COLUMN s.c FROM r1", + "ALTER TABLE t MODIFY COLUMN c STRING FROM r1", + "ALTER TABLE t MODIFY COLUMN s.c STRING FROM r1", + "ALTER TABLE t ORDER BY (c1, c2) FROM r1")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Rollup is not supported for Iceberg column operations")); + } + } + + @Test + void testRejectPropertiesForIcebergColumnOperations() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN c STRING NULL PROPERTIES ('k' = 'v')", + "ALTER TABLE t ADD COLUMN s.c STRING NULL PROPERTIES ('k' = 'v')", + "ALTER TABLE t ADD COLUMN (c1 STRING NULL, c2 INT NULL) PROPERTIES ('k' = 'v')", + "ALTER TABLE t DROP COLUMN c PROPERTIES ('k' = 'v')", + "ALTER TABLE t DROP COLUMN s.c PROPERTIES ('k' = 'v')", + "ALTER TABLE t MODIFY COLUMN c STRING PROPERTIES ('k' = 'v')", + "ALTER TABLE t MODIFY COLUMN s.c STRING PROPERTIES ('k' = 'v')", + "ALTER TABLE t ORDER BY (c1, c2) PROPERTIES ('k' = 'v')")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("PROPERTIES are not supported for Iceberg column operations")); + } + } + + @Test + void testRejectKeyForIcebergAddAndModify() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN c INT KEY NULL", + "ALTER TABLE t ADD COLUMN s.c INT KEY NULL", + "ALTER TABLE t ADD COLUMN (c1 INT KEY NULL, c2 INT NULL)", + "ALTER TABLE t MODIFY COLUMN c BIGINT KEY", + "ALTER TABLE t MODIFY COLUMN s.c BIGINT KEY")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("KEY is not supported for Iceberg ADD/MODIFY COLUMN")); + } + } + + @Test + void testRejectGeneratedColumnForIcebergAddAndModify() { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN c INT AS (id + 1) NULL", + "ALTER TABLE t ADD COLUMN s.c INT AS (id + 1) NULL", + "ALTER TABLE t ADD COLUMN (c1 INT AS (id + 1) NULL, c2 INT NULL)", + "ALTER TABLE t MODIFY COLUMN c BIGINT AS (id + 1)", + "ALTER TABLE t MODIFY COLUMN s.c BIGINT AS (id + 1)")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Generated columns are not supported for Iceberg ADD/MODIFY COLUMN")); + } + } + + @Test + void testRejectUnsupportedDefaultChangesForIcebergTable() throws AnalysisException { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t MODIFY COLUMN c BIGINT DEFAULT 7", + "ALTER TABLE t MODIFY COLUMN c BIGINT DEFAULT NULL", + "ALTER TABLE t MODIFY COLUMN ts DATETIME DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Modifying default values is not supported for Iceberg columns")); + } + + org.apache.doris.nereids.exceptions.AnalysisException complexDefaultException = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter( + "ALTER TABLE t MODIFY COLUMN st STRUCT DEFAULT 'x'").getOps())); + Assertions.assertTrue(complexDefaultException.getMessage() + .contains("Struct type column default value just support null")); + + AnalysisException onUpdateException = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter( + "ALTER TABLE t ADD COLUMN ts DATETIME NULL DEFAULT CURRENT_TIMESTAMP " + + "ON UPDATE CURRENT_TIMESTAMP").getOps())); + Assertions.assertTrue(onUpdateException.getMessage() + .contains("ON UPDATE is not supported for Iceberg ADD COLUMN")); + + AlterTableCommand.checkColumnOperationsSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN c BIGINT NULL DEFAULT 7").getOps()); + } + + @Test + void testRejectCompoundIcebergColumnOperations() throws AnalysisException { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN s.good INT NULL, DROP COLUMN m.value.x", + "ALTER TABLE t MODIFY COLUMN c COMMENT 'new comment', ADD COLUMN d INT NULL", + "ALTER TABLE t ADD COLUMN c INT NULL, DROP COLUMN d", + "ALTER TABLE t RENAME COLUMN c TO c2, RENAME COLUMN d TO d2", + "ALTER TABLE t MODIFY COLUMN c COMMENT 'c', MODIFY COLUMN d COMMENT 'd'", + "ALTER TABLE t ORDER BY (c, d), ORDER BY (d, c)")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> AlterTableCommand.checkColumnOperationsSupported(table, parseAlter(sql).getOps())); + Assertions.assertTrue(exception.getMessage() + .contains("Multiple Iceberg ALTER clauses are not supported")); + } + + AlterTableCommand.checkColumnOperationsSupported(table, + parseAlter("ALTER TABLE t ADD COLUMN (c1 INT NULL, c2 BIGINT NULL)").getOps()); + } + + @Test + void testPreserveEmptyAddColumnsValidationForIcebergTable() throws AnalysisException { + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + AddColumnsOp addColumnsOp = new AddColumnsOp(null, null, new HashMap<>()); + + AlterTableCommand.checkColumnOperationsSupported(table, Arrays.asList(addColumnsOp)); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> addColumnsOp.validate(null)); + Assertions.assertTrue(exception.getMessage().contains("Columns is empty in add columns clause")); + } + + @Test + void testModifyColumnTracksExplicitNullability() { + ModifyColumnOp omitted = (ModifyColumnOp) parseAlter( + "ALTER TABLE t MODIFY COLUMN s.a BIGINT").getOps().get(0); + ModifyColumnOp nullable = (ModifyColumnOp) parseAlter( + "ALTER TABLE t MODIFY COLUMN s.a BIGINT NULL").getOps().get(0); + ModifyColumnOp notNullable = (ModifyColumnOp) parseAlter( + "ALTER TABLE t MODIFY COLUMN s.a BIGINT NOT NULL").getOps().get(0); + + Assertions.assertFalse(omitted.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertTrue(nullable.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + Assertions.assertTrue(notNullable.getColumnDef() + .translateToCatalogStyleForSchemaChange().isNullableSpecified()); + } + + @Test + void testNestedIcebergColumnNamesBypassTopLevelSystemPrefixes() throws Exception { + Env env = Mockito.mock(Env.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + CatalogIf catalog = Mockito.mock(CatalogIf.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + IcebergExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalogOrDdlException("iceberg")).thenReturn(catalog); + Mockito.when(catalog.getDbOrDdlException("db")).thenReturn(database); + Mockito.when(database.getTableOrDdlException("t")).thenReturn(table); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN s.__DORIS_metric INT NULL", + "ALTER TABLE t ADD COLUMN s.__doris_shadow_metric INT NULL", + "ALTER TABLE t MODIFY COLUMN s.__DORIS_metric BIGINT", + "ALTER TABLE t MODIFY COLUMN s.__doris_shadow_metric BIGINT", + "ALTER TABLE t RENAME COLUMN s.__DORIS_metric TO metric", + "ALTER TABLE t RENAME COLUMN s.metric TO __doris_shadow_metric", + "ALTER TABLE t ADD COLUMN s._row_id BIGINT NULL", + "ALTER TABLE t MODIFY COLUMN s._row_id BIGINT", + "ALTER TABLE t RENAME COLUMN s.metric TO _last_updated_sequence_number", + "ALTER TABLE t DROP COLUMN s.__DORIS_metric")) { + validateIcebergAlter(sql, table); + } + + for (String sql : Arrays.asList( + "ALTER TABLE t ADD COLUMN __DORIS_metric INT NULL", + "ALTER TABLE t ADD COLUMN __doris_shadow_metric INT NULL", + "ALTER TABLE t MODIFY COLUMN __DORIS_metric BIGINT")) { + org.apache.doris.nereids.exceptions.AnalysisException exception = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> validateIcebergAlter(sql, table)); + Assertions.assertTrue(exception.getMessage().contains("Incorrect column name")); + } + + AnalysisException renameException = Assertions.assertThrows(AnalysisException.class, + () -> validateIcebergAlter( + "ALTER TABLE t RENAME COLUMN metric TO __DORIS_metric", table)); + Assertions.assertTrue(renameException.getMessage().contains("Incorrect column name")); + AnalysisException dropException = Assertions.assertThrows(AnalysisException.class, + () -> validateIcebergAlter("ALTER TABLE t DROP COLUMN __DORIS_metric", table)); + Assertions.assertTrue(dropException.getMessage().contains("Do not support drop hidden column")); + } + } + + private void validateIcebergAlter(String sql, IcebergExternalTable table) throws Exception { + AlterTableCommand command = parseAlter(sql); + AlterTableCommand.checkColumnOperationsSupported(table, command.getOps()); + for (AlterTableOp op : command.getOps()) { + op.setTableName(new TableNameInfo("iceberg", "db", "t")); + op.validate(null); + } + } + + private AlterTableCommand parseAlter(String sql) { + Plan plan = parser.parseSingle(sql); + Assertions.assertInstanceOf(AlterTableCommand.class, plan); + return (AlterTableCommand) plan; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommandTest.java new file mode 100644 index 00000000000000..6cd5d35d39d54c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommandTest.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.nereids.exceptions.AnalysisException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collections; + +public class DeleteFromCommandTest { + + @Test + public void testBuildDeleteFallbackExceptionPreservesBothFailureCauses() throws Exception { + DeleteFromCommand command = new DeleteFromCommand(Collections.emptyList(), null, + false, Collections.emptyList(), null); + Exception initialException = new Exception("initial predicate failure"); + Exception fallbackException = new Exception("fallback execution failure"); + + AnalysisException mergedException = invokeBuildDeleteFallbackException(command, + initialException, fallbackException); + + // Verify the merged exception surfaces the fallback failure and keeps the initial failure. + Assertions.assertEquals( + "Delete fallback execution failed: fallback execution failure" + + ". Initial predicate check failed: initial predicate failure", + mergedException.getMessage()); + Assertions.assertSame(fallbackException, mergedException.getCause()); + Assertions.assertEquals(1, mergedException.getSuppressed().length); + Assertions.assertSame(initialException, mergedException.getSuppressed()[0]); + } + + @Test + public void testBuildDeleteFallbackExceptionFallsBackToThrowableToString() throws Exception { + DeleteFromCommand command = new DeleteFromCommand(Collections.emptyList(), null, + false, Collections.emptyList(), null); + Exception initialException = new Exception((String) null); + Exception fallbackException = new Exception((String) null); + + AnalysisException mergedException = invokeBuildDeleteFallbackException(command, + initialException, fallbackException); + + // Verify null messages still produce debuggable text. + Assertions.assertEquals( + "Delete fallback execution failed: java.lang.Exception" + + ". Initial predicate check failed: java.lang.Exception", + mergedException.getMessage()); + Assertions.assertSame(fallbackException, mergedException.getCause()); + Assertions.assertEquals(1, mergedException.getSuppressed().length); + Assertions.assertSame(initialException, mergedException.getSuppressed()[0]); + } + + // Use reflection to validate the helper without exposing it only for tests. + private AnalysisException invokeBuildDeleteFallbackException(DeleteFromCommand command, + Exception initialException, Exception fallbackException) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Method method = DeleteFromCommand.class.getDeclaredMethod("buildDeleteFallbackException", + Exception.class, Exception.class); + method.setAccessible(true); + return (AnalysisException) method.invoke(command, initialException, fallbackException); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowRoutineLoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowRoutineLoadCommandTest.java index 4bbe272203040a..ce933be3decb99 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowRoutineLoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowRoutineLoadCommandTest.java @@ -35,5 +35,7 @@ public void testValidate() { LabelNameInfo labelNameInfo = new LabelNameInfo("test_db", "test_label"); ShowRoutineLoadCommand command = new ShowRoutineLoadCommand(labelNameInfo, null, false); Assertions.assertDoesNotThrow(() -> command.validate(connectContext)); + Assertions.assertEquals(24, command.getMetaData().getColumnCount()); + Assertions.assertEquals("FirstErrorMsg", command.getMetaData().getColumn(23).getName()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java index f20925011a4c44..6cbb4073df2d50 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.commands.info; +import org.apache.doris.nereids.types.StringType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -33,4 +35,12 @@ public void testNameEquals() { boolean expected2 = false; Assertions.assertEquals(expected2, columnDefinition.nameEquals(otherColName2, false)); } + + @Test + public void testToSqlHandlesNullComment() { + ColumnDefinition columnDefinition = new ColumnDefinition("col1", StringType.INSTANCE, true, null); + + String sql = columnDefinition.toSql(); + Assertions.assertTrue(sql.endsWith("COMMENT \"\"")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJobTest.java index db7f55fb2fd240..e5020e2b8a695e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJobTest.java @@ -20,9 +20,14 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector; import org.apache.doris.planner.DataPartition; +import org.apache.doris.planner.ExceptNode; import org.apache.doris.planner.ExchangeNode; +import org.apache.doris.planner.HashJoinNode; +import org.apache.doris.planner.IntersectNode; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.SetOperationNode; +import org.apache.doris.planner.UnionNode; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; import org.apache.doris.thrift.TUniqueId; @@ -34,6 +39,7 @@ import org.mockito.Mockito; import java.util.BitSet; +import java.util.List; public class UnassignedScanBucketOlapTableJobTest { @@ -114,4 +120,38 @@ public void testDegreeOfParallelismWithoutExchangeNodes() { int result = unassignedJob.degreeOfParallelism(3, false); Assertions.assertEquals(100, result); } + + @Test + public void testShouldFillUpInstancesSkipsBucketShuffleIntersectOnly() { + List noJoins = ImmutableList.of(); + + // A bucket-shuffle INTERSECT must NOT trigger missing-bucket receiver fill-up: a bucket the + // basic child does not scan is empty in the anchor, so the intersect result for that bucket + // is empty and the other children's rows shuffled there produce nothing. The skip must match + // the legacy planner IntersectNode; the translated node is never an instance of the Nereids + // algebra Intersect, so checking that type would silently never skip. + IntersectNode bucketShuffleIntersect = Mockito.mock(IntersectNode.class); + Mockito.when(bucketShuffleIntersect.isBucketShuffle()).thenReturn(true); + Assertions.assertFalse(UnassignedScanBucketOlapTableJob.shouldFillUpInstances( + noJoins, ImmutableList.of(bucketShuffleIntersect))); + + // Bucket-shuffle UNION / EXCEPT still need fill-up: the other children's rows shuffled into + // buckets the basic child does not scan must be received to be produced (union) or to + // subtract against (except) correctly. + UnionNode bucketShuffleUnion = Mockito.mock(UnionNode.class); + Mockito.when(bucketShuffleUnion.isBucketShuffle()).thenReturn(true); + Assertions.assertTrue(UnassignedScanBucketOlapTableJob.shouldFillUpInstances( + noJoins, ImmutableList.of(bucketShuffleUnion))); + + ExceptNode bucketShuffleExcept = Mockito.mock(ExceptNode.class); + Mockito.when(bucketShuffleExcept.isBucketShuffle()).thenReturn(true); + Assertions.assertTrue(UnassignedScanBucketOlapTableJob.shouldFillUpInstances( + noJoins, ImmutableList.of(bucketShuffleExcept))); + + // A union that did not choose bucket shuffle does not trigger fill-up. + UnionNode plainUnion = Mockito.mock(UnionNode.class); + Mockito.when(plainUnion.isBucketShuffle()).thenReturn(false); + Assertions.assertFalse(UnassignedScanBucketOlapTableJob.shouldFillUpInstances( + noJoins, ImmutableList.of(plainUnion))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java index 865bba61e1f3ad..ac0c76f80907b9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Type; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; @@ -36,6 +37,27 @@ public class LogicalFileScanTest { + @Test + public void testNestedColumnPruningForIcebergSystemTables() { + IcebergSysExternalTable positionDeletes = Mockito.mock(IcebergSysExternalTable.class); + Mockito.when(positionDeletes.initSelectedPartitions(Mockito.any())) + .thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(positionDeletes.isPositionDeletesTable()).thenReturn(true); + LogicalFileScan positionDeletesScan = new LogicalFileScan(new RelationId(1), positionDeletes, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + Assertions.assertTrue(positionDeletesScan.supportPruneNestedColumn()); + + IcebergSysExternalTable jniSystemTable = Mockito.mock(IcebergSysExternalTable.class); + Mockito.when(jniSystemTable.initSelectedPartitions(Mockito.any())) + .thenReturn(SelectedPartitions.NOT_PRUNED); + Mockito.when(jniSystemTable.isPositionDeletesTable()).thenReturn(false); + LogicalFileScan jniSystemTableScan = new LogicalFileScan(new RelationId(2), jniSystemTable, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + Assertions.assertFalse(jniSystemTableScan.supportPruneNestedColumn()); + } + @Test public void testComputeOutputIncludesInvisibleRowLineageColumnsForIcebergTable() { Column rowIdColumn = new Column(IcebergUtils.ICEBERG_ROW_ID_COL, Type.BIGINT, true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java index 23dcb4403ce731..df9b2ae80c26f2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/IcebergMergeSinkTest.java @@ -74,6 +74,63 @@ public void testBindDataSinkSkipsRewritableDeleteFileSetsAndRowLineageSchemaForV IcebergUtils.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL)); } + @Test + public void testBindDataSinkDisablesColumnStatsWhenAllMetricsAreNone() throws Exception { + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, Map.of( + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")), new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertFalse(thriftSink.isCollectColumnStats()); + } + + @Test + public void testBindDataSinkKeepsColumnStatsForMetricsOverride() throws Exception { + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, Map.of( + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "counts")), + new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertTrue(thriftSink.isCollectColumnStats()); + } + + @Test + public void testBindDataSinkKeepsColumnStatsForV3LineageFields() throws Exception { + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(3, Map.of( + TableProperties.DEFAULT_WRITE_METRICS_MODE, "counts", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "none")), + new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertTrue(thriftSink.isCollectColumnStats()); + } + + @Test + public void testBindDataSinkKeepsColumnStatsForOrcTopLevelComplexField() throws Exception { + Schema schema = new Schema(Types.NestedField.optional(1, "items", + Types.ListType.ofOptional(2, Types.IntegerType.get()))); + IcebergMergeSink sink = new IcebergMergeSink(mockIcebergExternalTable(2, schema, Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "orc", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "items", "counts")), + new DeleteCommandContext()); + + sink.bindDataSink(Optional.empty()); + + TIcebergMergeSink thriftSink = sink.tDataSink.getIcebergMergeSink(); + Assertions.assertTrue(thriftSink.isSetCollectColumnStats()); + Assertions.assertTrue(thriftSink.isCollectColumnStats()); + } + private static TIcebergRewritableDeleteFileSet buildDeleteFileSet() { TIcebergDeleteFileDesc deleteFileDesc = new TIcebergDeleteFileDesc(); deleteFileDesc.setPath("file:///tmp/delete.puffin"); @@ -84,13 +141,24 @@ private static TIcebergRewritableDeleteFileSet buildDeleteFileSet() { } private static IcebergExternalTable mockIcebergExternalTable(int formatVersion) { - Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + return mockIcebergExternalTable(formatVersion, Collections.emptyMap()); + } + + private static IcebergExternalTable mockIcebergExternalTable( + int formatVersion, Map metricsProperties) { + return mockIcebergExternalTable(formatVersion, + new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())), metricsProperties); + } + + private static IcebergExternalTable mockIcebergExternalTable( + int formatVersion, Schema schema, Map metricsProperties) { PartitionSpec spec = PartitionSpec.unpartitioned(); Map properties = new HashMap<>(); properties.put(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)); properties.put(TableProperties.DEFAULT_FILE_FORMAT, "parquet"); properties.put(TableProperties.PARQUET_COMPRESSION, "snappy"); properties.put(TableProperties.WRITE_DATA_LOCATION, "file:///tmp/iceberg_tbl/data"); + properties.putAll(metricsProperties); Table icebergTable = Mockito.mock(Table.class); Mockito.when(icebergTable.properties()).thenReturn(properties); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java index 5ffe61ce8f578e..e171ebe166ce2b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/LocalShuffleNodeCoverageTest.java @@ -53,6 +53,33 @@ public class LocalShuffleNodeCoverageTest { private static final AtomicInteger NEXT_ID = new AtomicInteger(1); + @Test + public void testRequireSpecificAutoRequireHashPreservesSpecificHash() { + // Pass-through operators (union / streaming agg / sort) forward their parent's specific + // hash requirement downward via autoRequireHash() while leaving row placement to their + // children. Every hash flavour must be forwarded unchanged: degrading a specific + // LOCAL_EXECUTION_HASH_SHUFFLE requirement to the generic RequireHash would let a + // bucket-distributed child satisfy it and keep its bucket placement while the operator + // still advertised LOCAL_EXECUTION_HASH_SHUFFLE upward, so a bucket join upgraded to + // local hash above it would skip its realign local exchange and compute wrong results. + for (LocalExchangeType hashType : new LocalExchangeType[] { + LocalExchangeType.LOCAL_EXECUTION_HASH_SHUFFLE, + LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE, + LocalExchangeType.BUCKET_HASH_SHUFFLE}) { + LocalExchangeNode.RequireSpecific require = new LocalExchangeNode.RequireSpecific(hashType); + LocalExchangeTypeRequire forwarded = require.autoRequireHash(); + Assertions.assertSame(require, forwarded, + "specific hash require " + hashType + " must be forwarded unchanged"); + Assertions.assertEquals(hashType, forwarded.preferType()); + } + + // A non-hash specific require still relaxes to the generic RequireHash, whose preferType + // is GLOBAL_EXECUTION_HASH_SHUFFLE. + LocalExchangeTypeRequire relaxed = + new LocalExchangeNode.RequireSpecific(LocalExchangeType.PASSTHROUGH).autoRequireHash(); + Assertions.assertEquals(LocalExchangeType.GLOBAL_EXECUTION_HASH_SHUFFLE, relaxed.preferType()); + } + @Test public void testSelectNode() { PlanTranslatorContext ctx = new PlanTranslatorContext(); @@ -613,6 +640,28 @@ public void testSetOperationAndAssertNumRowsNode() { Assertions.assertSame(exceptLeft, exceptNode.getChild(0)); Assertions.assertSame(exceptRight, exceptNode.getChild(1)); + // Bucket-shuffle IntersectNode (not colocated): exercises the `|| isBucketShuffle()` leg. + // Every child is distributed by the basic child's storage bucket function (via bucket-shuffle + // exchanges), so the output is BUCKET_HASH_SHUFFLE and each serial (NOOP) child is re-aligned + // with a BUCKET_HASH_SHUFFLE local exchange. Without the isBucketShuffle() branch this would + // fall into the partitioned (GLOBAL hash) leg and re-partition one side by a different hash + // function, breaking alignment. setColocate(false) keeps isColocated() false (SetOperationNode + // returns false immediately when isColocate() is false), so the branch is reached through + // isBucketShuffle() alone. + IntersectNode bucketIntersect = new IntersectNode(nextPlanNodeId(), + new TupleId(NEXT_ID.getAndIncrement())); + bucketIntersect.setColocate(false); + bucketIntersect.setDistributionMode(DistributionMode.BUCKET_SHUFFLE); + TrackingPlanNode bucketLeft = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP); + TrackingPlanNode bucketRight = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP); + bucketIntersect.addChild(bucketLeft); + bucketIntersect.addChild(bucketRight); + Pair bucketIntersectOutput = bucketIntersect.enforceAndDeriveLocalExchange( + ctx, null, LocalExchangeTypeRequire.requireHash()); + Assertions.assertEquals(LocalExchangeType.BUCKET_HASH_SHUFFLE, bucketIntersectOutput.second); + assertChildLocalExchangeType(bucketIntersect, 0, LocalExchangeType.BUCKET_HASH_SHUFFLE); + assertChildLocalExchangeType(bucketIntersect, 1, LocalExchangeType.BUCKET_HASH_SHUFFLE); + TrackingPlanNode assertChild = new TrackingPlanNode(nextPlanNodeId(), LocalExchangeType.NOOP); AssertNumRowsElement assertElement = Mockito.mock(AssertNumRowsElement.class); Mockito.when(assertElement.getDesiredNumOfRows()).thenReturn(1L); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java index e8bf8b628c3171..d7cb0e7d94c434 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java @@ -117,7 +117,29 @@ protected void runBeforeAll() throws Exception { + "distributed by hash(k1) buckets 3\n" + "properties('replication_num' = '1')"; - createTables(nonPart, part1, part2, multiLeveParts, variantTable); + String uniqueMowTable = "create table db1.uniq_mow(" + + " k1 int,\n" + + " v1 int)\n" + + "UNIQUE KEY(k1)\n" + + "distributed by hash(k1) buckets 3\n" + + "properties('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true')"; + + String uniqueMorTable = "create table db1.uniq_mor(" + + " k1 int,\n" + + " v1 int)\n" + + "UNIQUE KEY(k1)\n" + + "distributed by hash(k1) buckets 3\n" + + "properties('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'false')"; + + String aggTable = "create table db1.agg_tbl(" + + " k1 int,\n" + + " v1 int sum)\n" + + "AGGREGATE KEY(k1)\n" + + "distributed by hash(k1) buckets 3\n" + + "properties('replication_num' = '1')"; + + createTables(nonPart, part1, part2, multiLeveParts, variantTable, uniqueMowTable, + uniqueMorTable, aggTable); connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); connectContext.getSessionVariable().setEnableQueryCache(true); @@ -382,6 +404,72 @@ public void testVariantSubColumnDigest() throws Exception { Assertions.assertEquals(digest1, digest3); } + @Test + public void testAllowIncremental() throws Exception { + // Switch off (the default): never allow incremental merge. + Assertions.assertFalse(connectContext.getSessionVariable().getEnableQueryCacheIncremental()); + TQueryCacheParam withoutSwitch = getQueryCacheParam( + "select k2, sum(v1) as v from db1.part1 group by k2"); + Assertions.assertFalse(withoutSwitch.allow_incremental); + + connectContext.getSessionVariable().setEnableQueryCacheIncremental(true); + try { + // DUP_KEYS with a two-phase aggregation (group by a non-distribution + // column): the cache point is the non-finalize first phase, whose + // output is merged again upstream, so incremental merge is safe. + TQueryCacheParam dupTwoPhase = getQueryCacheParam( + "select k2, sum(v1) as v from db1.part1 group by k2"); + Assertions.assertTrue(dupTwoPhase.allow_incremental); + + // One-phase aggregation finalizes inside the cached fragment: its + // output has no downstream merge, so emitting cached and delta + // blocks side by side would duplicate group keys. + TQueryCacheParam onePhase = phaseAgg(1, () -> getQueryCacheParam( + "select k1, sum(v1) as v from db1.non_part group by k1")); + Assertions.assertFalse(onePhase.allow_incremental); + + // UNIQUE merge-on-write is append-only as long as loads do not + // touch pre-existing keys; FE grants the capability and BE checks + // the delete bitmap of the delta window per tablet. Group by a + // non-distribution column so the aggregation stays two-phase and + // the decision reaches the keys-type check. + TQueryCacheParam uniqueMow = getQueryCacheParam( + "select v1, count(*) as v from db1.uniq_mow group by v1"); + Assertions.assertTrue(uniqueMow.allow_incremental); + + // UNIQUE merge-on-read resolves duplicates by merging across + // rowsets at read time: a delta-only scan cannot stand alone. + TQueryCacheParam uniqueMor = getQueryCacheParam( + "select v1, count(*) as v from db1.uniq_mor group by v1"); + Assertions.assertFalse(uniqueMor.allow_incremental); + + // AGG_KEYS merges rows inside the storage layer. + TQueryCacheParam aggKeys = getQueryCacheParam( + "select v1, count(*) as v from db1.agg_tbl group by v1"); + Assertions.assertFalse(aggKeys.allow_incremental); + + // Agg over Agg inside one fragment (distinct aggregation grouped by + // the distribution column): the cache point is not directly on the + // scan (and finalizes here as well), so incremental is not allowed. + TQueryCacheParam distinctAgg = getQueryCacheParam( + "select k1, count(distinct k2) from db1.non_part group by k1"); + Assertions.assertFalse(distinctAgg.allow_incremental); + + // Nested cache point: a non-finalize partial agg over a finalized + // colocate agg over scan. The inner agg would see only the delta + // rows during an incremental run, so its finalized output is not a + // mergeable complement of the cached snapshot -- must be rejected + // even though the cache point itself does not finalize. + TQueryCacheParam nestedAgg = getQueryCacheParam( + "select cnt, count(*) from" + + " (select k1, count(*) cnt from db1.non_part group by k1) x" + + " group by cnt"); + Assertions.assertFalse(nestedAgg.allow_incremental); + } finally { + connectContext.getSessionVariable().setEnableQueryCacheIncremental(false); + } + } + private String getDigest(String sql) throws Exception { return Hex.encodeHexString(getQueryCacheParam(sql).digest); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index 94d922e69876a8..509d1823afc415 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -753,4 +753,62 @@ public void testConnectAttributesSetNull() { Assert.assertNotNull(ctx.getConnectAttributes()); Assert.assertTrue(ctx.getConnectAttributes().isEmpty()); } + + // Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259). + // closeFlightSqlDeferredExecutors() is the single place that releases those deferred coordinators + // (and with them the external-table batch SplitSource and the query queue slot). The following + // tests pin the leak-prevention contract of that method: every deferred executor is finalized, + // the list is cleared so nothing is finalized twice or retained, and one failing executor does + // not strand the others' resources. + + @Test + public void testCloseFlightSqlDeferredExecutorsFinalizesEachExecutor() { + ConnectContext ctx = new ConnectContext(); + StmtExecutor deferred1 = Mockito.mock(StmtExecutor.class); + StmtExecutor deferred2 = Mockito.mock(StmtExecutor.class); + ctx.addFlightSqlDeferredExecutor(deferred1); + ctx.addFlightSqlDeferredExecutor(deferred2); + + ctx.closeFlightSqlDeferredExecutors(); + + // Both deferred coordinators must be finalized, otherwise their SplitSource and query queue + // slot leak after the DoGet phase. + Mockito.verify(deferred1).finalizeArrowFlightQuery(); + Mockito.verify(deferred2).finalizeArrowFlightQuery(); + } + + @Test + public void testCloseFlightSqlDeferredExecutorsClearsListSoSecondCallIsNoOp() { + ConnectContext ctx = new ConnectContext(); + StmtExecutor deferred = Mockito.mock(StmtExecutor.class); + ctx.addFlightSqlDeferredExecutor(deferred); + + // More than one teardown path can fire for the same connection (e.g. the next query cleans + // up, then the connection is later torn down). The list must be cleared after the first + // call so the executor is finalized exactly once and is not retained (leaked) afterwards. + ctx.closeFlightSqlDeferredExecutors(); + ctx.closeFlightSqlDeferredExecutors(); + + Mockito.verify(deferred, Mockito.times(1)).finalizeArrowFlightQuery(); + } + + @Test + public void testCloseFlightSqlDeferredExecutorsFinalizesRemainingWhenOneFails() { + ConnectContext ctx = new ConnectContext(); + StmtExecutor failing = Mockito.mock(StmtExecutor.class); + StmtExecutor healthy = Mockito.mock(StmtExecutor.class); + Mockito.doThrow(new RuntimeException("finalize failed")).when(failing).finalizeArrowFlightQuery(); + ctx.addFlightSqlDeferredExecutor(failing); + ctx.addFlightSqlDeferredExecutor(healthy); + + // A single bad coordinator must not abort the cleanup: the call must not throw, and the + // healthy executor must still be finalized so its resources are released. + ctx.closeFlightSqlDeferredExecutors(); + Mockito.verify(healthy).finalizeArrowFlightQuery(); + + // The list is cleared up front, so neither executor is reprocessed on a later teardown. + ctx.closeFlightSqlDeferredExecutors(); + Mockito.verify(failing, Mockito.times(1)).finalizeArrowFlightQuery(); + Mockito.verify(healthy, Mockito.times(1)).finalizeArrowFlightQuery(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java index 32055e16b865c5..75bc69ba7c5531 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java @@ -79,6 +79,39 @@ public void testForwardSessionVariables() { sessionVariable.getInsertVisibleTimeoutReturnModeEnum()); } + @Test + public void testForwardQueryCacheVariables() { + // A forwarded statement is planned by the master in a fresh + // ConnectContext that only sees what getForwardVariables() sends, so + // every session variable the query cache reads at plan time must + // travel: the switch and its incremental companion (both planner gates + // and the eligibility check), and the three the cache param carries + // (a forced refresh must not be dropped, and entries must be sized by + // the session's limits, not the master's defaults). + SessionVariable follower = new SessionVariable(); + follower.setEnableQueryCache(true); + follower.setEnableQueryCacheIncremental(true); + follower.setQueryCacheForceRefresh(true); + follower.setQueryCacheEntryMaxBytes(4096); + follower.setQueryCacheEntryMaxRows(64); + Map vars = follower.getForwardVariables(); + Assertions.assertEquals("true", vars.get(SessionVariable.ENABLE_QUERY_CACHE)); + Assertions.assertEquals("true", vars.get(SessionVariable.ENABLE_QUERY_CACHE_INCREMENTAL)); + Assertions.assertEquals("true", vars.get(SessionVariable.QUERY_CACHE_FORCE_REFRESH)); + Assertions.assertEquals("4096", vars.get(SessionVariable.QUERY_CACHE_ENTRY_MAX_BYTES)); + Assertions.assertEquals("64", vars.get(SessionVariable.QUERY_CACHE_ENTRY_MAX_ROWS)); + + SessionVariable master = new SessionVariable(); + Assertions.assertFalse(master.getEnableQueryCache()); + Assertions.assertFalse(master.getEnableQueryCacheIncremental()); + master.setForwardedSessionVariables(vars); + Assertions.assertTrue(master.getEnableQueryCache()); + Assertions.assertTrue(master.getEnableQueryCacheIncremental()); + Assertions.assertTrue(master.isQueryCacheForceRefresh()); + Assertions.assertEquals(4096, master.getQueryCacheEntryMaxBytes()); + Assertions.assertEquals(64, master.getQueryCacheEntryMaxRows()); + } + @Test public void testInsertVisibleTimeoutReturnMode() throws Exception { connectContext.setThreadLocalInfo(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorInternalQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorInternalQueryTest.java index a167c1b3adedf4..ee2b87d1abf6f6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorInternalQueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorInternalQueryTest.java @@ -18,12 +18,17 @@ package org.apache.doris.qe; import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.Env; import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.resource.workloadschedpolicy.WorkloadRuntimeStatusMgr; import org.apache.doris.thrift.TQueryOptions; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.MockedConstruction; import org.mockito.Mockito; @@ -33,6 +38,7 @@ public void testSetSqlHash() { StmtExecutor executor = new StmtExecutor(new ConnectContext(), "select * from table1"); try (MockedConstruction mocked = Mockito.mockConstruction(NereidsPlanner.class, (mock, context) -> { + Mockito.when(mock.getStatementContext()).thenReturn(executor.getContext().getStatementContext()); Mockito.doThrow(new RuntimeException()).when(mock).plan( Mockito.any(StatementBase.class), Mockito.any(TQueryOptions.class)); })) { @@ -53,8 +59,11 @@ public void testExecuteInternalQuerySetsErrorStateOnFailure() { ConnectContext ctx = new ConnectContext(); StmtExecutor executor = new StmtExecutor(ctx, "select * from table1"); try (MockedConstruction mocked = Mockito.mockConstruction(NereidsPlanner.class, - (mock, context) -> Mockito.doThrow(new RuntimeException("mock plan failure")) - .when(mock).plan(Mockito.any(StatementBase.class), Mockito.any(TQueryOptions.class)))) { + (mock, context) -> { + Mockito.when(mock.getStatementContext()).thenReturn(ctx.getStatementContext()); + Mockito.doThrow(new RuntimeException("mock plan failure")) + .when(mock).plan(Mockito.any(StatementBase.class), Mockito.any(TQueryOptions.class)); + })) { Assert.assertThrows(RuntimeException.class, executor::executeInternalQuery); } Assert.assertEquals(QueryState.MysqlStateType.ERR, ctx.getState().getStateType()); @@ -67,4 +76,43 @@ public void testExecuteInternalQuerySetsErrorStateOnFailure() { Assert.assertTrue("internal query should be flagged as query in audit state", ctx.getState().isQuery()); } + + @Test + public void testExecuteInternalQuerySubmitsErrorAuditEventOnFailure() { + ConnectContext ctx = new ConnectContext(); + StmtExecutor executor = new StmtExecutor(ctx, "select * from table1"); + Env env = Env.getCurrentEnv(); + WorkloadRuntimeStatusMgr originalWorkloadRuntimeStatusMgr = env.getWorkloadRuntimeStatusMgr(); + WorkloadRuntimeStatusMgr workloadRuntimeStatusMgr = Mockito.mock(WorkloadRuntimeStatusMgr.class); + ArgumentCaptor auditEventCaptor = ArgumentCaptor.forClass(AuditEvent.class); + + Deencapsulation.setField(env, "workloadRuntimeStatusMgr", workloadRuntimeStatusMgr); + try { + try (MockedConstruction mockedPlanner = Mockito.mockConstruction(NereidsPlanner.class, + (mock, context) -> { + Mockito.when(mock.getStatementContext()).thenReturn(ctx.getStatementContext()); + Mockito.doThrow(new RuntimeException("mock plan failure")) + .when(mock).plan(Mockito.any(StatementBase.class), Mockito.any(TQueryOptions.class)); + })) { + Assert.assertThrows(RuntimeException.class, executor::executeInternalQuery); + } + + Mockito.verify(workloadRuntimeStatusMgr).submitFinishQueryToAudit(auditEventCaptor.capture()); + } finally { + Deencapsulation.setField(env, "workloadRuntimeStatusMgr", originalWorkloadRuntimeStatusMgr); + } + + AuditEvent auditEvent = auditEventCaptor.getValue(); + Assert.assertEquals(AuditEvent.EventType.AFTER_QUERY, auditEvent.type); + Assert.assertEquals("ERR", auditEvent.state); + Assert.assertEquals(ErrorCode.ERR_INTERNAL_ERROR.getCode(), auditEvent.errorCode); + Assert.assertNotNull(auditEvent.errorMessage); + Assert.assertTrue("error message should mention root cause, got: " + auditEvent.errorMessage, + auditEvent.errorMessage.contains("mock plan failure")); + Assert.assertTrue("audit event should be marked as internal", auditEvent.isInternal); + Assert.assertTrue("audit event should be marked as query", auditEvent.isQuery); + Assert.assertTrue("audit event should be marked as nereids", auditEvent.isNereids); + Assert.assertEquals("select * from table1", auditEvent.stmt); + Assert.assertEquals("a8ec30e5ad0820f8c5bd16a82a4491ca", auditEvent.sqlHash); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index b7b12bc91c3deb..607cea3b40a302 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -29,6 +29,8 @@ import org.apache.doris.planner.ResultFileSink; import org.apache.doris.qe.CommonResultSet.CommonResultSetMetaData; import org.apache.doris.qe.ConnectContext.ConnectType; +import org.apache.doris.thrift.TQueryOptions; +import org.apache.doris.thrift.TUniqueId; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.Lists; @@ -71,6 +73,40 @@ public void testShowNull() throws Exception { Assert.assertEquals(QueryState.MysqlStateType.OK, connectContext.getState().getStateType()); } + // Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259); + // it is released later by finalizeArrowFlightQuery(), which closes the coordinator and then + // unregisters the query. The close and the unregister must be independent: if coord.close() + // throws, the query registration must still be released (the try/finally), otherwise the query + // leaks in QeProcessorImpl forever. The thrown error is expected to propagate to the caller + // (ConnectContext.closeFlightSqlDeferredExecutors), which catches and logs it. + @Test + public void testFinalizeArrowFlightQueryUnregistersQueryEvenIfCoordCloseThrows() throws Exception { + StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); + TUniqueId queryId = new TUniqueId(0x6226259L, 0x62259L); + connectContext.setQueryId(queryId); + + Coordinator coord = Mockito.mock(Coordinator.class); + Mockito.when(coord.getQueryOptions()).thenReturn(new TQueryOptions()); + Mockito.doThrow(new RuntimeException("coord close failed")).when(coord).close(); + stmtExecutor.setCoord(coord); + + // Simulate the in-flight query whose results DoGet is still pulling. + QeProcessorImpl.INSTANCE.registerQuery(queryId, new QeProcessorImpl.QueryInfo(coord)); + Assert.assertNotNull(QeProcessorImpl.INSTANCE.getCoordinator(queryId)); + + try { + stmtExecutor.finalizeArrowFlightQuery(); + Assert.fail("expected coord.close() failure to propagate after the query is unregistered"); + } catch (RuntimeException e) { + Assert.assertEquals("coord close failed", e.getMessage()); + } + + // The coordinator close was attempted (releases SplitSource + query queue slot) ... + Mockito.verify(coord).close(); + // ... and despite it failing, the query registration was still released (no leak). + Assert.assertNull(QeProcessorImpl.INSTANCE.getCoordinator(queryId)); + } + @Test public void testKill() throws Exception { StmtExecutor stmtExecutor = new StmtExecutor(connectContext, ""); diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplCloudTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplCloudTest.java new file mode 100644 index 00000000000000..a5a28080315ca9 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplCloudTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.service; + +import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.CacheHotspotManager; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.common.Config; +import org.apache.doris.thrift.TGetTabletReplicaInfosRequest; +import org.apache.doris.thrift.TGetTabletReplicaInfosResult; +import org.apache.doris.thrift.TStatusCode; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; + +public class FrontendServiceImplCloudTest { + + // Regression test for FrontendServiceImpl.getTabletReplicaInfos NPE: + // When a warm-up job has been removed from + // CacheHotspotManager.cloudWarmUpJobs (past + // history_cloud_warm_up_job_keep_max_second), getCloudWarmUpJob + // returns null. The previous code called job.getJobId() inside the + // log message, throwing NPE which bubbled up to BE as + // "Internal error processing getTabletReplicaInfos". + @Test + public void testGetTabletReplicaInfosNullJobReturnsCancelledWithoutNpe() { + String originalCloudUniqueId = Config.cloud_unique_id; + Config.cloud_unique_id = "gettabletreplicainfostest"; + + CloudEnv cloudEnv = Mockito.mock(CloudEnv.class); + CacheHotspotManager cacheHotspotManager = Mockito.mock(CacheHotspotManager.class); + Mockito.when(cloudEnv.getCacheHotspotMgr()).thenReturn(cacheHotspotManager); + // Simulate job already removed from cloudWarmUpJobs. + Mockito.when(cacheHotspotManager.getCloudWarmUpJob(123456L)).thenReturn(null); + + try (MockedStatic envMock = Mockito.mockStatic(Env.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(cloudEnv); + + FrontendServiceImpl frontendService = new FrontendServiceImpl(Mockito.mock(ExecuteEnv.class)); + TGetTabletReplicaInfosRequest request = new TGetTabletReplicaInfosRequest(); + request.setTabletIds(Collections.singletonList(789L)); + request.setWarmUpJobId(123456L); + + TGetTabletReplicaInfosResult result; + try { + result = frontendService.getTabletReplicaInfos(request); + } catch (NullPointerException e) { + throw new AssertionError("getTabletReplicaInfos must not NPE when the " + + "warm-up job has been removed from CacheHotspotManager", e); + } + + Assert.assertNotNull("result.status must be set", result.getStatus()); + Assert.assertEquals("BE must be told to cancel its stale warm-up job entry", + TStatusCode.CANCELLED, result.getStatus().getStatusCode()); + } finally { + Config.cloud_unique_id = originalCloudUniqueId; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java index 008c10d91df0b4..87cb3f8e2c3452 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplTest.java @@ -23,8 +23,6 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; -import org.apache.doris.cloud.CacheHotspotManager; -import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.common.AuthenticationException; import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; @@ -51,8 +49,6 @@ import org.apache.doris.thrift.TGetDbsResult; import org.apache.doris.thrift.TGetTablesParams; import org.apache.doris.thrift.TGetTablesResult; -import org.apache.doris.thrift.TGetTabletReplicaInfosRequest; -import org.apache.doris.thrift.TGetTabletReplicaInfosResult; import org.apache.doris.thrift.TListTableStatusResult; import org.apache.doris.thrift.TLoadTxnCommitRequest; import org.apache.doris.thrift.TLoadTxnRollbackRequest; @@ -92,6 +88,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; public class FrontendServiceImplTest { @@ -376,6 +373,54 @@ public void testCreatePartitionRange() throws Exception { Assert.assertNotNull(p20230807); } + @Test + public void testCreatePartitionReturnsRetryErrorWhenResultPartitionIsMissing() throws Exception { + String createOlapTblStmt = "CREATE TABLE test.partition_dropped_before_result_snapshot(\n" + + " event_day DATETIME NOT NULL,\n" + + " site_id INT\n" + + ")\n" + + "DUPLICATE KEY(event_day, site_id)\n" + + "AUTO PARTITION BY RANGE (date_trunc(event_day, 'day')) ()\n" + + "DISTRIBUTED BY HASH(event_day) BUCKETS 1\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; + createTable(createOlapTblStmt); + + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("test"); + OlapTable table = (OlapTable) db.getTableOrAnalysisException("partition_dropped_before_result_snapshot"); + OlapTable spyTable = Mockito.spy(table); + String partitionName = "p20230808000000"; + AtomicBoolean hideResultPartition = new AtomicBoolean(true); + Mockito.doAnswer(invocation -> { + Partition partition = (Partition) invocation.callRealMethod(); + // Model the lookup result after a concurrent retention drop without timing-dependent test threads. + if (partition != null && hideResultPartition.compareAndSet(true, false)) { + return null; + } + return partition; + }).when(spyTable).getPartition(partitionName); + + db.unregisterTable(table.getName()); + db.registerTable(spyTable); + try { + TNullableStringLiteral start = new TNullableStringLiteral(); + start.setValue("2023-08-08 00:00:00"); + TCreatePartitionRequest request = new TCreatePartitionRequest(); + request.setDbId(db.getId()); + request.setTableId(spyTable.getId()); + request.setPartitionValues(Collections.singletonList(Collections.singletonList(start))); + + TCreatePartitionResult result = new FrontendServiceImpl(exeEnv).createPartition(request); + + Assert.assertEquals(TStatusCode.RUNTIME_ERROR, result.getStatus().getStatusCode()); + Assert.assertTrue(result.getStatus().getErrorMsgs().get(0) + .contains("was dropped concurrently while building auto partition result, please retry")); + Assert.assertFalse(hideResultPartition.get()); + } finally { + db.unregisterTable(spyTable.getName()); + db.registerTable(table); + } + } + @Test public void testCreatePartitionList() throws Exception { String createOlapTblStmt = new String("CREATE TABLE test.partition_list(\n" @@ -732,50 +777,6 @@ public void testRollbackTxnRejectsInvalidToken() { } } - // Regression test for FrontendServiceImpl.getTabletReplicaInfos NPE: - // When a warm-up job has been removed from - // CacheHotspotManager.cloudWarmUpJobs (past - // history_cloud_warm_up_job_keep_max_second), getCloudWarmUpJob - // returns null. The previous code called job.getJobId() inside the - // log message, throwing NPE which bubbled up to BE as - // "Internal error processing getTabletReplicaInfos". - @Test - public void testGetTabletReplicaInfosNullJobReturnsCancelledWithoutNpe() { - String originalCloudUniqueId = Config.cloud_unique_id; - Config.cloud_unique_id = "gettabletreplicainfostest"; - - CloudEnv cloudEnv = Mockito.mock(CloudEnv.class); - CacheHotspotManager cacheHotspotManager = Mockito.mock(CacheHotspotManager.class); - Mockito.when(cloudEnv.getCacheHotspotMgr()).thenReturn(cacheHotspotManager); - // Simulate job already removed from cloudWarmUpJobs. - Mockito.when(cacheHotspotManager.getCloudWarmUpJob(123456L)).thenReturn(null); - - MockedStatic envMock = Mockito.mockStatic(Env.class); - try { - envMock.when(Env::getCurrentEnv).thenReturn(cloudEnv); - - FrontendServiceImpl frontendService = new FrontendServiceImpl(exeEnv); - TGetTabletReplicaInfosRequest request = new TGetTabletReplicaInfosRequest(); - request.setTabletIds(Collections.singletonList(789L)); - request.setWarmUpJobId(123456L); - - TGetTabletReplicaInfosResult result; - try { - result = frontendService.getTabletReplicaInfos(request); - } catch (NullPointerException e) { - throw new AssertionError("getTabletReplicaInfos must not NPE when the " - + "warm-up job has been removed from CacheHotspotManager", e); - } - - Assert.assertNotNull("result.status must be set", result.getStatus()); - Assert.assertEquals("BE must be told to cancel its stale warm-up job entry", - TStatusCode.CANCELLED, result.getStatus().getStatusCode()); - } finally { - envMock.close(); - Config.cloud_unique_id = originalCloudUniqueId; - } - } - private MockedStatic transactionValidationEnvMock; private void mockTransactionForTokenValidation(long txnId) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java new file mode 100644 index 00000000000000..eede12688517be --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducerTest.java @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.service.arrowflight; + +import org.apache.doris.common.FeConstants; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.service.arrowflight.results.FlightSqlChannel; +import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager; +import org.apache.doris.service.arrowflight.sessions.FlightSqlConnectContext; + +import org.apache.arrow.flight.FlightDescriptor; +import org.apache.arrow.flight.FlightProducer.CallContext; +import org.apache.arrow.flight.FlightProducer.StreamListener; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.Result; +import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementRequest; +import org.apache.arrow.flight.sql.impl.FlightSql.CommandStatementQuery; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockedConstruction; +import org.mockito.Mockito; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class DorisFlightSqlProducerTest { + + private boolean prevRunningUnitTest; + + @Before + public void setUp() { + // FlightSqlConnectContext.init() only reaches Env when this is false; keep it true so the + // context can be built without a running FE. + prevRunningUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = true; + } + + @After + public void tearDown() { + FeConstants.runningUnitTest = prevRunningUnitTest; + } + + /** + * Regression test for the FE direct-memory leak in {@code createPreparedStatement} + * (issue apache/doris#65305, fixed by PR #65311). + * + *

Before the fix, each prepare allocated a {@link org.apache.arrow.vector.VectorSchemaRoot} + * via {@code FlightSqlChannel.createOneOneSchemaRoot("ResultMeta", ...)} and read only its + * {@code Schema}, never closing the root, leaking one off-heap {@code VarCharVector} buffer per + * prepare. Because Arrow Flight has no parameter binding, every client query triggers a fresh + * prepare, so the leak grows monotonically until {@code MaxDirectMemorySize} is exhausted. + * + *

This test drives {@code createPreparedStatement} against a real {@link FlightSqlChannel} many + * times and asserts the channel's Arrow allocator holds zero bytes afterwards. It goes red if the + * try-with-resources that closes the temporary roots is removed. + */ + @Test + public void createPreparedStatementDoesNotLeakChannelAllocator() throws Exception { + // A real flight session context owns a real FlightSqlChannel (and thus a real Arrow allocator), + // so allocator bookkeeping is exercised for real instead of mocked away. + FlightSqlConnectContext connectContext = new FlightSqlConnectContext("test-peer-identity"); + FlightSqlChannel channel = connectContext.getFlightSqlChannel(); + Assert.assertEquals("channel allocator should start empty", 0L, channel.getAllocatedMemory()); + + FlightSessionsManager sessionsManager = new FlightSessionsManager() { + @Override + public ConnectContext getConnectContext(String peerIdentity) { + return connectContext; + } + + @Override + public ConnectContext createConnectContext(String peerIdentity) { + return connectContext; + } + + @Override + public void closeConnectContext(String peerIdentity) { + // not exercised by this test + } + }; + DorisFlightSqlProducer producer = + new DorisFlightSqlProducer(Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager); + + CallContext callContext = Mockito.mock(CallContext.class); + Mockito.when(callContext.peerIdentity()).thenReturn("test-peer-identity"); + + final int rounds = 100; + AtomicInteger errors = new AtomicInteger(0); + try { + // createPreparedStatement runs asynchronously and mutates a shared, non-thread-safe + // ConnectContext, so drive it serially: each prepare completes before the next is issued. + // The leak, if any, still accumulates on the single channel allocator across rounds. + for (int i = 0; i < rounds; i++) { + CountDownLatch finished = new CountDownLatch(1); + StreamListener listener = new StreamListener() { + @Override + public void onNext(Result val) { + // discard the placeholder prepared-statement result + } + + @Override + public void onError(Throwable t) { + errors.incrementAndGet(); + finished.countDown(); + } + + @Override + public void onCompleted() { + finished.countDown(); + } + }; + ActionCreatePreparedStatementRequest request = ActionCreatePreparedStatementRequest.newBuilder() + .setQuery("select * from t where id = " + i).build(); + producer.createPreparedStatement(request, callContext, listener); + Assert.assertTrue("createPreparedStatement #" + i + " did not finish in time", + finished.await(30, TimeUnit.SECONDS)); + } + + // Guard against a false pass: if a prepare failed before reaching the allocation, no buffer + // would be leaked and the memory assertion below could not detect a regression. + Assert.assertEquals("no createPreparedStatement call should fail", 0, errors.get()); + // Every temporary VectorSchemaRoot must have been closed, so the channel's Arrow allocator + // is back to zero. Reverting the fix leaves `rounds` ResultMeta buffers allocated here. + Assert.assertEquals("createPreparedStatement leaked off-heap memory in the channel allocator", + 0L, channel.getAllocatedMemory()); + } finally { + producer.close(); + } + } + + // Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259): + // executeAndSendResult() registers it as a deferred executor on the ConnectContext right after + // submitting it to the BE. GetFlightInfo then still has to fetch the Arrow schema from the BE. + // If that fetch fails (timeout / non-OK / empty / mismatched schema / RPC error), no FlightInfo + // is returned, so no DoGet will ever pull this query's results. The deferred coordinator must be + // finalized on this error path; otherwise its external-table batch SplitSource, query queue slot + // and query registration leak until the next query starts or the connection is torn down. + @Test + public void testGetFlightInfoFinalizesDeferredExecutorWhenSchemaFetchFails() throws Exception { + // A flight ConnectContext whose getFlightSqlChannel() works (the base context throws). + ConnectContext ctx = Mockito.spy(new ConnectContext()); + Mockito.doReturn(Mockito.mock(FlightSqlChannel.class)).when(ctx).getFlightSqlChannel(); + + // Stands in for the just-planned external-table query whose results DoGet would pull from BE. + StmtExecutor deferred = Mockito.mock(StmtExecutor.class); + + FlightSessionsManager sessionsManager = Mockito.mock(FlightSessionsManager.class); + Mockito.when(sessionsManager.getConnectContext(Mockito.anyString())).thenReturn(ctx); + + CallContext callContext = Mockito.mock(CallContext.class); + Mockito.when(callContext.peerIdentity()).thenReturn("token"); + + DorisFlightSqlProducer producer = new DorisFlightSqlProducer( + Location.forGrpcInsecure("127.0.0.1", 9090), sessionsManager); + try (MockedConstruction mocked = Mockito.mockConstruction( + FlightSqlConnectProcessor.class, (mock, context) -> { + // handleQuery plans + submits to BE and defers the coordinator (coordBase == coord), + // exactly as executeAndSendResult() does for an Arrow Flight external-table scan. + Mockito.doAnswer(invocation -> { + ctx.setReturnResultFromLocal(false); + ctx.addFlightSqlDeferredExecutor(deferred); + return null; + }).when(mock).handleQuery(Mockito.anyString()); + // The Arrow schema fetch fails after the coordinator was already deferred. + Mockito.doThrow(new RuntimeException("fetch arrow flight schema timeout")) + .when(mock).fetchArrowFlightSchema(Mockito.anyInt()); + })) { + CommandStatementQuery request = CommandStatementQuery.newBuilder().setQuery("select 1").build(); + FlightDescriptor descriptor = FlightDescriptor.command(new byte[0]); + + try { + producer.getFlightInfoStatement(request, callContext, descriptor); + Assert.fail("expected the schema fetch failure to propagate as a CallStatus"); + } catch (Throwable expected) { + // GetFlightInfo is expected to fail; the point of the test is what happens to the + // deferred coordinator, not the thrown status itself. + } + + // The deferred coordinator of the failed query is finalized on the error path instead of + // leaking until the next query / connection teardown. + Mockito.verify(deferred).finalizeArrowFlightQuery(); + + // It is also removed from the deferred list, so a later teardown does not finalize it + // again (no double-close, no retained reference). + ctx.closeFlightSqlDeferredExecutors(); + Mockito.verify(deferred, Mockito.times(1)).finalizeArrowFlightQuery(); + } finally { + producer.close(); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java new file mode 100644 index 00000000000000..80dcb5b4346e12 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/sessions/FlightSqlConnectPoolMgrTest.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.service.arrowflight.sessions; + +import org.apache.doris.qe.ConnectContext; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class FlightSqlConnectPoolMgrTest { + + // Arrow Flight SQL keeps a query's coordinator alive across GetFlightInfo -> DoGet (see #62259). + // unregisterConnection() is the catch-all teardown path: idle/query timeout, bearer token expiry + // and explicit CloseSession all reach here. It must finalize the deferred coordinators so an + // abandoned connection cannot leak them (the external-table batch SplitSource and the query + // queue slot the coordinator holds). + @Test + public void testUnregisterConnectionFinalizesDeferredExecutors() { + FlightSqlConnectPoolMgr poolMgr = new FlightSqlConnectPoolMgr(100); + ConnectContext ctx = Mockito.mock(ConnectContext.class); + + poolMgr.unregisterConnection(ctx); + + // The deferred coordinators must be released on teardown even though this connection was + // never registered in the pool (an abandoned connection is still cleaned up, not leaked). + Mockito.verify(ctx).closeFlightSqlDeferredExecutors(); + } + + // Cleanup must run before the connection bookkeeping (closeTxn / map removal), so that a failure + // there cannot strand the deferred coordinators. Verify the deferred executors are finalized + // even when the context is the one stored in the pool. + @Test + public void testUnregisterRegisteredConnectionFinalizesDeferredExecutors() { + FlightSqlConnectPoolMgr poolMgr = new FlightSqlConnectPoolMgr(100); + ConnectContext ctx = Mockito.mock(ConnectContext.class); + Mockito.when(ctx.getConnectionId()).thenReturn(7); + Mockito.when(ctx.getConnectType()).thenReturn(ConnectContext.ConnectType.ARROW_FLIGHT_SQL); + Mockito.when(ctx.getPeerIdentity()).thenReturn("token-7"); + poolMgr.getConnectionMap().put(7, ctx); + + poolMgr.unregisterConnection(ctx); + + Mockito.verify(ctx).closeFlightSqlDeferredExecutors(); + Assert.assertNull(poolMgr.getConnectionMap().get(7)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/query/QueryStatsRecorderTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/query/QueryStatsRecorderTest.java index 7d747f55f572a4..2dbf6da6dd294f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/query/QueryStatsRecorderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/query/QueryStatsRecorderTest.java @@ -24,6 +24,8 @@ import org.apache.doris.common.Config; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -31,15 +33,28 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalExcept; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; +import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveUnionAnchor; import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalWorkTableReference; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.QueryState; @@ -563,8 +578,9 @@ public void testAliasUnwrappedForQueryHitIntermediateProject() { StatsDelta delta = deltas.get("1_1_1_1"); Assertions.assertNotNull(delta); Assertions.assertTrue(delta.getColumnStats().get("k2").queryHit, "k2: ORDER BY"); - Assertions.assertNotNull(delta.getColumnStats().get("x"), "x slot resolves via alias propagation"); - Assertions.assertTrue(delta.getColumnStats().get("x").queryHit, "x: SELECT output via alias"); + Assertions.assertNotNull(delta.getColumnStats().get("k1"), + "k1 resolves via alias propagation under intermediate Sort"); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit, "k1: SELECT output via alias"); } /** @@ -600,6 +616,135 @@ public void testGroupByAndAggregateInputQueryHit() { Assertions.assertTrue(delta.getColumnStats().get("k2").queryHit, "k2: aggregate input"); } + /** + * SELECT DISTINCT UPPER(k6) ... ORDER BY UPPER(k6): the merge phase of two-phase aggregation + * outputs a bare pass-through SlotReference reusing the local phase's ExprId. Regression test + * for a StackOverflowError this used to cause (getInputSlots() on a bare Slot includes itself). + */ + @Test + @SuppressWarnings("unchecked") + public void testTwoPhaseAggregateBarePassthroughOutputDoesNotSelfReference() { + ExprId id6 = new ExprId(1); + ExprId mergeOutputId = new ExprId(2); + SlotReference k6Slot = mockSlot(id6, "k6"); + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k6Slot)); + + // Local (partial) phase: real Alias(Upper(k6), Y) — links Y -> k6. + NamedExpression localOutput = Mockito.mock(NamedExpression.class); + Mockito.when(localOutput.getExprId()).thenReturn(mergeOutputId); + Mockito.when(localOutput.getInputSlots()).thenReturn(ImmutableSet.of(k6Slot)); + + Aggregate localAgg = Mockito.mock(Aggregate.class); + Mockito.when(localAgg.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(localAgg.getGroupByExpressions()).thenReturn(ImmutableList.of()); + Mockito.when(localAgg.getOutputExpressions()) + .thenReturn((List) (List) ImmutableList.of(localOutput)); + Mockito.when(localAgg.getOutput()).thenReturn(ImmutableList.of()); + + // Merge (final) phase: bare pass-through SlotReference reusing the SAME ExprId Y. + // getInputSlots() includes itself, matching real Expression.getInputSlots() for a leaf Slot. + SlotReference mergeOutputSlot = mockSlot(mergeOutputId, "upper_k6"); + Mockito.when(mergeOutputSlot.getInputSlots()).thenReturn(ImmutableSet.of(mergeOutputSlot)); + + Aggregate mergeAgg = Mockito.mock(Aggregate.class); + Mockito.when(mergeAgg.children()).thenReturn(ImmutableList.of(localAgg)); + Mockito.when(mergeAgg.getGroupByExpressions()).thenReturn(ImmutableList.of()); + Mockito.when(mergeAgg.getOutputExpressions()) + .thenReturn((List) (List) ImmutableList.of(mergeOutputSlot)); + Mockito.when(mergeAgg.getOutput()).thenReturn(ImmutableList.of(mergeOutputSlot)); + + // ORDER BY the merge phase's own output slot directly (same identity, no recomputation). + Expression orderInner = Mockito.mock(Expression.class); + Mockito.when(orderInner.getInputSlots()).thenReturn(ImmutableSet.of(mergeOutputSlot)); + org.apache.doris.nereids.properties.OrderKey orderKey = + Mockito.mock(org.apache.doris.nereids.properties.OrderKey.class); + Mockito.when(orderKey.getExpr()).thenReturn(orderInner); + + org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort sort = + Mockito.mock(org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort.class); + Mockito.when(sort.children()).thenReturn(ImmutableList.of(mergeAgg)); + Mockito.when(sort.getOrderKeys()).thenReturn(ImmutableList.of(orderKey)); + Mockito.when(sort.getOutput()).thenReturn(ImmutableList.of(mergeOutputSlot)); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) sort); + + StatsDelta delta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(delta); + Assertions.assertTrue(delta.getColumnStats().get("k6").queryHit, "k6: resolves via the local phase's link"); + } + + /** + * Regression for the `visiting` cycle guard's generality: a true 2-hop cycle (derivedSlotInputs + * entry A points to B, and B's entry points back to A — no real scan column involved at all, + * unlike the length-1 self-loop test above). Two stacked PhysicalProject nodes each alias the + * other's ExprId, forming the cycle; a real k1 column exists separately so collectDeltas + * doesn't short-circuit on an empty exprIdToScan. Must terminate with no attribution for the + * cyclic slot, not StackOverflowError, and must not disturb resolution of the unrelated k1. + */ + @Test + @SuppressWarnings("unchecked") + public void testMultiHopCycleInDerivedSlotInputsDoesNotOverflow() { + ExprId k1Id = new ExprId(1); + ExprId aId = new ExprId(100); + ExprId bId = new ExprId(101); + SlotReference k1Slot = mockSlot(k1Id, "k1"); + SlotReference slotA = mockSlot(aId, "a_ref"); + SlotReference slotB = mockSlot(bId, "b_ref"); + + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Slot)); + + // Project1: Alias(SlotRef(a_ref)) with exprId B — links B -> {slotA}. + org.apache.doris.nereids.trees.expressions.Alias alias1 = + Mockito.mock(org.apache.doris.nereids.trees.expressions.Alias.class); + Mockito.when(alias1.getExprId()).thenReturn(bId); + Mockito.when(alias1.child()).thenReturn(slotA); + + PhysicalProject project1 = Mockito.mock(PhysicalProject.class); + Mockito.when(project1.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(project1.getProjects()).thenReturn(ImmutableList.of(alias1, k1Slot)); + Mockito.when(project1.getOutput()).thenReturn(ImmutableList.of()); + + // Project2: Alias(SlotRef(b_ref)) with exprId A — links A -> {slotB}, closing the cycle. + org.apache.doris.nereids.trees.expressions.Alias alias2 = + Mockito.mock(org.apache.doris.nereids.trees.expressions.Alias.class); + Mockito.when(alias2.getExprId()).thenReturn(aId); + Mockito.when(alias2.child()).thenReturn(slotB); + + PhysicalProject project2 = Mockito.mock(PhysicalProject.class); + Mockito.when(project2.children()).thenReturn(ImmutableList.of(project1)); + Mockito.when(project2.getProjects()).thenReturn(ImmutableList.of(alias2)); + Mockito.when(project2.getOutput()).thenReturn(ImmutableList.of()); + + // ORDER BY the cyclic slot A, plus a real ORDER BY on k1 to prove the cycle doesn't + // disturb unrelated resolution. + Expression cyclicOrderExpr = Mockito.mock(Expression.class); + Mockito.when(cyclicOrderExpr.getInputSlots()).thenReturn(ImmutableSet.of(slotA)); + org.apache.doris.nereids.properties.OrderKey cyclicOrderKey = + Mockito.mock(org.apache.doris.nereids.properties.OrderKey.class); + Mockito.when(cyclicOrderKey.getExpr()).thenReturn(cyclicOrderExpr); + + Expression k1OrderExpr = Mockito.mock(Expression.class); + Mockito.when(k1OrderExpr.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot)); + org.apache.doris.nereids.properties.OrderKey k1OrderKey = + Mockito.mock(org.apache.doris.nereids.properties.OrderKey.class); + Mockito.when(k1OrderKey.getExpr()).thenReturn(k1OrderExpr); + + org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort sort = + Mockito.mock(org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort.class); + Mockito.when(sort.children()).thenReturn(ImmutableList.of(project2)); + Mockito.when(sort.getOrderKeys()).thenReturn(ImmutableList.of(cyclicOrderKey, k1OrderKey)); + Mockito.when(sort.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = Assertions.assertDoesNotThrow( + () -> QueryStatsRecorder.collectDeltas((PhysicalPlan) sort), + "a 2-hop cycle in derivedSlotInputs must not cause a StackOverflowError"); + + StatsDelta delta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(delta); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit, + "k1: unrelated resolution must still work correctly alongside the cycle"); + } + /** * SELECT k1 FROM t ORDER BY k2: ORDER BY k2 → queryHit. */ @@ -845,6 +990,150 @@ public void testWindowFunctionValueColumnQueryHit() { Assertions.assertTrue(delta.getColumnStats().get("k2").queryHit, "k2: SUM value column"); } + /** + * QUALIFY-style filter above a window on a value-preserving window function: + * SELECT k1 FROM (SELECT k1, SUM(k2) OVER (PARTITION BY k0 ORDER BY k1) AS running_sum + * FROM t) w WHERE running_sum > 100. The window alias must be linked to its function's + * own inputs so the filter resolves back to k2, the same way HAVING resolves through an + * aggregate output. + */ + @Test + @SuppressWarnings("unchecked") + public void testFilterAboveWindowValueColumnRecordsFilterHit() { + ExprId id0 = new ExprId(1); + ExprId id1 = new ExprId(2); + ExprId id2 = new ExprId(3); + ExprId windowAliasId = new ExprId(10); + + SlotReference k0Slot = mockSlot(id0, "k0"); + SlotReference k1Slot = mockSlot(id1, "k1"); + SlotReference k2Slot = mockSlot(id2, "k2"); + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, + ImmutableList.of(k0Slot, k1Slot, k2Slot)); + + // Window function SUM(k2) — its input slots include k2 + Expression sumFunc = Mockito.mock(Expression.class); + Mockito.when(sumFunc.getInputSlots()).thenReturn(ImmutableSet.of(k2Slot)); + + WindowExpression windowExpr = Mockito.mock(WindowExpression.class); + Mockito.when(windowExpr.getFunction()).thenReturn(sumFunc); + Mockito.when(windowExpr.getInputSlots()).thenReturn(ImmutableSet.of(k0Slot, k1Slot, k2Slot)); + + NamedExpression windowAlias = Mockito.mock(NamedExpression.class); + Mockito.when(windowAlias.getExprId()).thenReturn(windowAliasId); + Mockito.when(windowAlias.child(0)).thenReturn(windowExpr); + + Expression partExpr = Mockito.mock(Expression.class); + Mockito.when(partExpr.getInputSlots()).thenReturn(ImmutableSet.of(k0Slot)); + + OrderExpression orderExpr = Mockito.mock(OrderExpression.class); + Expression orderInner = Mockito.mock(Expression.class); + Mockito.when(orderInner.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot)); + Mockito.when(orderExpr.child()).thenReturn(orderInner); + + org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup wfg = + Mockito.mock( + org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup.class); + Mockito.when(wfg.getPartitionKeys()).thenReturn(ImmutableSet.of(partExpr)); + Mockito.when(wfg.getOrderKeys()).thenReturn(ImmutableList.of(orderExpr)); + Mockito.when(wfg.getGroups()).thenReturn(ImmutableList.of(windowAlias)); + + org.apache.doris.nereids.trees.plans.physical.PhysicalWindow window = + Mockito.mock(org.apache.doris.nereids.trees.plans.physical.PhysicalWindow.class); + Mockito.when(window.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(window.getWindowFrameGroup()).thenReturn(wfg); + Mockito.when(window.getOutput()).thenReturn(ImmutableList.of()); + + // QUALIFY-style filter above the window, referencing the window alias's own output. + SlotReference windowOutputSlot = mockSlot(windowAliasId, "running_sum"); + Expression filterConjunct = Mockito.mock(Expression.class); + Mockito.when(filterConjunct.getInputSlots()).thenReturn(ImmutableSet.of(windowOutputSlot)); + + PhysicalFilter filter = Mockito.mock(PhysicalFilter.class); + Mockito.when(filter.children()).thenReturn(ImmutableList.of(window)); + Mockito.when(filter.getConjuncts()).thenReturn(ImmutableSet.of(filterConjunct)); + Mockito.when(filter.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) filter); + + StatsDelta delta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(delta); + Assertions.assertTrue(delta.getColumnStats().get("k2").filterHit, + "k2: filter above window on SUM(k2) OVER(...) value column"); + } + + /** + * QUALIFY-style filter above ROW_NUMBER() OVER (PARTITION BY k0 ORDER BY k1): the function + * itself takes no arguments, so function.getInputSlots() is empty and cannot link the alias + * to anything — the alias must instead link through the FULL WindowExpression (which wires + * the partition/order keys as children too), so the filter still resolves to k0/k1. + */ + @Test + @SuppressWarnings("unchecked") + public void testFilterAbovePositionalWindowFunctionRecordsFilterHit() { + ExprId id0 = new ExprId(1); + ExprId id1 = new ExprId(2); + ExprId windowAliasId = new ExprId(10); + + SlotReference k0Slot = mockSlot(id0, "k0"); + SlotReference k1Slot = mockSlot(id1, "k1"); + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k0Slot, k1Slot)); + + // ROW_NUMBER() — no arguments, getInputSlots() is empty. + Expression rowNumberFunc = Mockito.mock(Expression.class); + Mockito.when(rowNumberFunc.getInputSlots()).thenReturn(ImmutableSet.of()); + + WindowExpression windowExpr = Mockito.mock(WindowExpression.class); + Mockito.when(windowExpr.getFunction()).thenReturn(rowNumberFunc); + // The WindowExpression's OWN getInputSlots() still includes partition/order keys even + // though the function itself has none — this is what the fix now links through. + Mockito.when(windowExpr.getInputSlots()).thenReturn(ImmutableSet.of(k0Slot, k1Slot)); + + NamedExpression windowAlias = Mockito.mock(NamedExpression.class); + Mockito.when(windowAlias.getExprId()).thenReturn(windowAliasId); + Mockito.when(windowAlias.child(0)).thenReturn(windowExpr); + + Expression partExpr = Mockito.mock(Expression.class); + Mockito.when(partExpr.getInputSlots()).thenReturn(ImmutableSet.of(k0Slot)); + + OrderExpression orderExpr = Mockito.mock(OrderExpression.class); + Expression orderInner = Mockito.mock(Expression.class); + Mockito.when(orderInner.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot)); + Mockito.when(orderExpr.child()).thenReturn(orderInner); + + org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup wfg = + Mockito.mock( + org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup.class); + Mockito.when(wfg.getPartitionKeys()).thenReturn(ImmutableSet.of(partExpr)); + Mockito.when(wfg.getOrderKeys()).thenReturn(ImmutableList.of(orderExpr)); + Mockito.when(wfg.getGroups()).thenReturn(ImmutableList.of(windowAlias)); + + org.apache.doris.nereids.trees.plans.physical.PhysicalWindow window = + Mockito.mock(org.apache.doris.nereids.trees.plans.physical.PhysicalWindow.class); + Mockito.when(window.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(window.getWindowFrameGroup()).thenReturn(wfg); + Mockito.when(window.getOutput()).thenReturn(ImmutableList.of()); + + // QUALIFY-style filter above the window, referencing the window alias's own output. + SlotReference windowOutputSlot = mockSlot(windowAliasId, "rn"); + Expression filterConjunct = Mockito.mock(Expression.class); + Mockito.when(filterConjunct.getInputSlots()).thenReturn(ImmutableSet.of(windowOutputSlot)); + + PhysicalFilter filter = Mockito.mock(PhysicalFilter.class); + Mockito.when(filter.children()).thenReturn(ImmutableList.of(window)); + Mockito.when(filter.getConjuncts()).thenReturn(ImmutableSet.of(filterConjunct)); + Mockito.when(filter.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) filter); + + StatsDelta delta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(delta); + Assertions.assertTrue(delta.getColumnStats().get("k0").filterHit, + "k0: QUALIFY on ROW_NUMBER() resolves through the partition key"); + Assertions.assertTrue(delta.getColumnStats().get("k1").filterHit, + "k1: QUALIFY on ROW_NUMBER() resolves through the order key"); + } + // ── helpers ────────────────────────────────────────────────────────────── @Test @@ -890,6 +1179,91 @@ public void testPhysicalRepeatRegistersGroupingExpressionsAsQueryHit() { Assertions.assertNull(delta.getColumnStats().get("k1"), "k1 not in grouping set — no hit"); } + /** + * SELECT k0, SUM(k2) FROM t GROUP BY ROLLUP(k0): k0 is a grouping key (getGroupingSets()), + * k2 feeds the aggregate function via a non-grouping output column (getOutputExpressions()). + * The existing repeat test never stubs getOutputExpressions(), leaving this loop untested. + */ + @Test + @SuppressWarnings("unchecked") + public void testPhysicalRepeatRegistersOutputExpressionsAsQueryHit() { + Config.enable_query_hit_stats = true; + ExprId id0 = new ExprId(0); + ExprId id2 = new ExprId(2); + SlotReference k0Slot = mockSlot(id0, "k0"); + SlotReference k2Slot = mockSlot(id2, "k2"); + PhysicalOlapScan scan = mockScan(1, 1, 1, 1, ImmutableList.of(k0Slot, k2Slot)); + + Expression groupExpr = Mockito.mock(Expression.class); + Mockito.when(groupExpr.getInputSlots()).thenReturn(ImmutableSet.of(k0Slot)); + + // SUM(k2): a non-grouping-key output column consumed by an aggregate function above. + NamedExpression sumExpr = Mockito.mock(NamedExpression.class); + Mockito.when(sumExpr.getInputSlots()).thenReturn(ImmutableSet.of(k2Slot)); + + PhysicalRepeat repeat = Mockito.mock(PhysicalRepeat.class); + Mockito.when(repeat.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(repeat.getOutput()).thenReturn(ImmutableList.of()); + Mockito.when(repeat.getGroupingSets()) + .thenReturn(ImmutableList.of(ImmutableList.of(groupExpr))); + Mockito.when(repeat.getOutputExpressions()) + .thenReturn((List) (List) ImmutableList.of(sumExpr)); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) repeat); + StatsDelta delta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(delta); + Assertions.assertTrue(delta.getColumnStats().get("k0").queryHit, "k0: ROLLUP grouping key"); + Assertions.assertTrue(delta.getColumnStats().get("k2").queryHit, + "k2: non-grouping output column feeding SUM(k2) under ROLLUP/CUBE"); + } + + /** + * SELECT a, ... FROM t GROUP BY ROLLUP(a) HAVING GROUPING(a) = 1: the repeat output + * Alias(Grouping(a), G) is not scan-backed and has no Alias wrapper stubbed here (matching + * how repeat.getOutputExpressions() elements are consumed directly) — its own ExprId (G) + * must link back to `a` so a HAVING filter on the grouping indicator resolves to filterHit. + */ + @Test + @SuppressWarnings("unchecked") + public void testPhysicalRepeatGroupingOutputRecordsHavingFilterHit() { + ExprId aId = new ExprId(0); + ExprId groupingId = new ExprId(2); + SlotReference aSlot = mockSlot(aId, "a"); + PhysicalOlapScan scan = mockScan(1, 1, 1, 1, ImmutableList.of(aSlot)); + + Expression groupExpr = Mockito.mock(Expression.class); + Mockito.when(groupExpr.getInputSlots()).thenReturn(ImmutableSet.of(aSlot)); + + // Alias(Grouping(a), G): a non-scan-backed output whose own getInputSlots() is {a}. + NamedExpression groupingExpr = Mockito.mock(NamedExpression.class); + Mockito.when(groupingExpr.getExprId()).thenReturn(groupingId); + Mockito.when(groupingExpr.getInputSlots()).thenReturn(ImmutableSet.of(aSlot)); + + PhysicalRepeat repeat = Mockito.mock(PhysicalRepeat.class); + Mockito.when(repeat.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(repeat.getOutput()).thenReturn(ImmutableList.of()); + Mockito.when(repeat.getGroupingSets()) + .thenReturn(ImmutableList.of(ImmutableList.of(groupExpr))); + Mockito.when(repeat.getOutputExpressions()) + .thenReturn((List) (List) ImmutableList.of(groupingExpr)); + + // HAVING GROUPING(a) = 1: filter references the repeat output's own ExprId directly. + SlotReference groupingSlot = mockSlot(groupingId, "grouping_a"); + Expression havingConjunct = Mockito.mock(Expression.class); + Mockito.when(havingConjunct.getInputSlots()).thenReturn(ImmutableSet.of(groupingSlot)); + + PhysicalFilter having = Mockito.mock(PhysicalFilter.class); + Mockito.when(having.children()).thenReturn(ImmutableList.of(repeat)); + Mockito.when(having.getConjuncts()).thenReturn(ImmutableSet.of(havingConjunct)); + Mockito.when(having.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) having); + StatsDelta delta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(delta); + Assertions.assertTrue(delta.getColumnStats().get("a").filterHit, + "a: HAVING GROUPING(a) = 1 resolves back through the repeat output's own lineage"); + } + @Test public void testPhysicalPartitionTopNRegistersPartitionAndOrderKeysAsQueryHit() { Config.enable_query_hit_stats = true; @@ -921,7 +1295,1000 @@ public void testPhysicalPartitionTopNRegistersPartitionAndOrderKeysAsQueryHit() Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit, "k1: ORDER BY in PartitionTopN"); } - // ── helpers ────────────────────────────────────────────────────────────── + // ── provenance resolver: multi-hop computed-slot chains ───────────────── + + /** + * SELECT k1+1 FROM t: single-input computed alias at the query root. + * Before the shared resolver, single-input aliases were registered directly in + * exprIdToScan by the PhysicalProject handler, but the root output loop's fallback + * only checked derivedSlotInputs — so this case silently recorded nothing. + * Expected: k1.queryHit=true. + */ + @Test + @SuppressWarnings("unchecked") + public void testSingleInputComputedRootSelectRecordsQueryHit() { + ExprId k1Id = new ExprId(1); + ExprId aliasId = new ExprId(99); + SlotReference k1Slot = mockSlot(k1Id, "k1"); + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Slot)); + + // Simulates k1+1 — one input slot; the literal contributes no slot. + Expression addExpr = Mockito.mock(Expression.class); + Mockito.when(addExpr.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot)); + + Alias alias = Mockito.mock(Alias.class); + Mockito.when(alias.getExprId()).thenReturn(aliasId); + Mockito.when(alias.child()).thenReturn(addExpr); + + PhysicalProject project = Mockito.mock(PhysicalProject.class); + Mockito.when(project.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(project.getProjects()).thenReturn(ImmutableList.of(alias)); + Mockito.when(project.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) project); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("k1"), + "k1 must be recorded from computed SELECT k1+1"); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit); + } + + /** + * WITH cte AS (SELECT k1+k2 AS x FROM t) SELECT x FROM cte: CTE consumer over a + * computed producer column. Before the shared resolver, the CTE consumer branch only + * copied a producer slot that was already a direct exprIdToScan entry — a computed + * producer column (in derivedSlotInputs) was silently skipped. + * Expected: k1.queryHit=true AND k2.queryHit=true. + */ + @Test + @SuppressWarnings("unchecked") + public void testCteConsumerOverComputedProducerColumnRecordsQueryHit() { + ExprId k1Id = new ExprId(1); + ExprId k2Id = new ExprId(2); + ExprId producerXId = new ExprId(10); + ExprId consumerXId = new ExprId(20); + + SlotReference k1Slot = mockSlot(k1Id, "k1"); + SlotReference k2Slot = mockSlot(k2Id, "k2"); + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Slot, k2Slot)); + + // Producer: SELECT k1+k2 AS x FROM t + Expression addExpr = Mockito.mock(Expression.class); + Mockito.when(addExpr.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot, k2Slot)); + Alias xAlias = Mockito.mock(Alias.class); + Mockito.when(xAlias.getExprId()).thenReturn(producerXId); + Mockito.when(xAlias.child()).thenReturn(addExpr); + + PhysicalProject producerProject = Mockito.mock(PhysicalProject.class); + Mockito.when(producerProject.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(producerProject.getProjects()).thenReturn(ImmutableList.of(xAlias)); + Mockito.when(producerProject.getOutput()).thenReturn(ImmutableList.of()); + + PhysicalCTEProducer producer = Mockito.mock(PhysicalCTEProducer.class); + Mockito.when(producer.children()).thenReturn(ImmutableList.of(producerProject)); + + SlotReference producerXSlot = mockSlot(producerXId, "x"); + SlotReference consumerXSlot = mockSlot(consumerXId, "x"); + + PhysicalCTEConsumer consumer = Mockito.mock(PhysicalCTEConsumer.class); + Mockito.when(consumer.getOutput()).thenReturn(ImmutableList.of(consumerXSlot)); + Mockito.when(consumer.getProducerSlot(consumerXSlot)).thenReturn(producerXSlot); + Mockito.when(consumer.children()).thenReturn(ImmutableList.of()); + + PhysicalFilter root = Mockito.mock(PhysicalFilter.class); + Mockito.when(root.children()).thenReturn(ImmutableList.of(producer, consumer)); + Mockito.when(root.getConjuncts()).thenReturn(ImmutableSet.of()); + Mockito.when(root.getOutput()).thenReturn(ImmutableList.of(consumerXSlot)); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) root); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("k1"), + "k1 must be recorded via CTE consumer over a computed producer column"); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit); + Assertions.assertNotNull(delta.getColumnStats().get("k2")); + Assertions.assertTrue(delta.getColumnStats().get("k2").queryHit); + } + + /** + * SELECT k1+k2 FROM t UNION ALL SELECT k1+k2 FROM t: a branch's contributing column is + * itself a computed alias. Before the shared resolver, recordSetOpChildrenOutputs only + * checked exprIdToScan directly and skipped computed branch outputs. + * Expected: k1.queryHit=true AND k2.queryHit=true. + */ + @Test + @SuppressWarnings("unchecked") + public void testSetOperationComputedBranchColumnRecordsQueryHit() { + ExprId k1Id = new ExprId(1); + ExprId k2Id = new ExprId(2); + ExprId branchAliasId = new ExprId(50); + + SlotReference k1Slot = mockSlot(k1Id, "k1"); + SlotReference k2Slot = mockSlot(k2Id, "k2"); + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Slot, k2Slot)); + + // Branch: SELECT k1+k2 FROM t + Expression addExpr = Mockito.mock(Expression.class); + Mockito.when(addExpr.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot, k2Slot)); + Alias branchAlias = Mockito.mock(Alias.class); + Mockito.when(branchAlias.getExprId()).thenReturn(branchAliasId); + Mockito.when(branchAlias.child()).thenReturn(addExpr); + + PhysicalProject branchProject = Mockito.mock(PhysicalProject.class); + Mockito.when(branchProject.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(branchProject.getProjects()).thenReturn(ImmutableList.of(branchAlias)); + Mockito.when(branchProject.getOutput()).thenReturn(ImmutableList.of()); + + // The set operation's own child-output slot at this position shares the alias's ExprId. + SlotReference branchOutputSlot = mockSlot(branchAliasId, "k1 + k2"); + + PhysicalSetOperation union = Mockito.mock(PhysicalSetOperation.class); + Mockito.when(union.children()).thenReturn(ImmutableList.of(branchProject)); + Mockito.when(union.getRegularChildrenOutputs()) + .thenReturn(ImmutableList.of(ImmutableList.of(branchOutputSlot))); + Mockito.when(union.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) union); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("k1"), + "k1 must be recorded via a computed UNION branch column"); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit); + Assertions.assertNotNull(delta.getColumnStats().get("k2")); + Assertions.assertTrue(delta.getColumnStats().get("k2").queryHit); + } + + /** + * SELECT SUM(x) FROM (SELECT k1+k2 AS x FROM t) s HAVING SUM(x) > 0: the aggregate's + * single input (x) is itself a computed column. Before the shared resolver, the + * single-input HAVING branch only checked exprIdToScan for that one input and never + * fell back to derivedSlotInputs, so a nested computed input was silently dropped. + * Expected: k1.filterHit=true AND k2.filterHit=true. + */ + @Test + @SuppressWarnings("unchecked") + public void testNestedSingleInputHavingOverComputedColumnRecordsFilterHit() { + ExprId k1Id = new ExprId(1); + ExprId k2Id = new ExprId(2); + ExprId xId = new ExprId(10); + ExprId sumXId = new ExprId(20); + + SlotReference k1Slot = mockSlot(k1Id, "k1"); + SlotReference k2Slot = mockSlot(k2Id, "k2"); + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Slot, k2Slot)); + + // Subquery: SELECT k1+k2 AS x FROM t + Expression addExpr = Mockito.mock(Expression.class); + Mockito.when(addExpr.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot, k2Slot)); + Alias xAlias = Mockito.mock(Alias.class); + Mockito.when(xAlias.getExprId()).thenReturn(xId); + Mockito.when(xAlias.child()).thenReturn(addExpr); + + PhysicalProject subqueryProject = Mockito.mock(PhysicalProject.class); + Mockito.when(subqueryProject.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(subqueryProject.getProjects()).thenReturn(ImmutableList.of(xAlias)); + Mockito.when(subqueryProject.getOutput()).thenReturn(ImmutableList.of()); + + SlotReference xSlot = mockSlot(xId, "x"); + + // Outer aggregate: SUM(x) — one input slot, but x is itself computed. + NamedExpression sumExpr = Mockito.mock(NamedExpression.class); + Mockito.when(sumExpr.getExprId()).thenReturn(sumXId); + Mockito.when(sumExpr.getInputSlots()).thenReturn(ImmutableSet.of(xSlot)); + + Aggregate agg = Mockito.mock(Aggregate.class); + Mockito.when(agg.children()).thenReturn(ImmutableList.of(subqueryProject)); + Mockito.when(agg.getGroupByExpressions()).thenReturn(ImmutableList.of()); + Mockito.when(agg.getOutputExpressions()) + .thenReturn((List) (List) ImmutableList.of(sumExpr)); + Mockito.when(agg.getOutput()).thenReturn(ImmutableList.of()); + + SlotReference sumSlot = mockSlot(sumXId, "sum_x"); + Expression havingConjunct = Mockito.mock(Expression.class); + Mockito.when(havingConjunct.getInputSlots()).thenReturn(ImmutableSet.of(sumSlot)); + + PhysicalFilter having = Mockito.mock(PhysicalFilter.class); + Mockito.when(having.children()).thenReturn(ImmutableList.of(agg)); + Mockito.when(having.getConjuncts()).thenReturn(ImmutableSet.of(havingConjunct)); + Mockito.when(having.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) having); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("k1"), + "k1 must get filterHit via HAVING SUM(x) where x is itself computed"); + Assertions.assertTrue(delta.getColumnStats().get("k1").filterHit); + Assertions.assertNotNull(delta.getColumnStats().get("k2")); + Assertions.assertTrue(delta.getColumnStats().get("k2").filterHit); + } + + /** + * SELECT * FROM (SELECT k1 FROM t1 UNION ALL SELECT k1 FROM t2) v WHERE k1 > 0, where + * the filter sits above the set operation and references its output slot directly (the + * shape PushDownFilterThroughSetOperation leaves in place for predicates it cannot push + * into a branch, e.g. involving a volatile function). Before the shared resolver, the + * set operation's own output ExprId was never linked to its branches, so a parent filter + * referencing it resolved to nothing. + * Expected: k1.filterHit=true on both underlying tables. + */ + @Test + public void testFilterAboveSetOperationResolvesToScanColumns() { + ExprId leftId = new ExprId(1); + ExprId rightId = new ExprId(3); + ExprId setOutputId = new ExprId(50); + + SlotReference k1Left = mockSlot(leftId, "k1"); + SlotReference k1Right = mockSlot(rightId, "k1"); + PhysicalOlapScan scan1 = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Left)); + PhysicalOlapScan scan2 = mockScan(1L, 2L, 2L, 1L, ImmutableList.of(k1Right)); + + SlotReference setOutputSlot = mockSlot(setOutputId, "k1"); + + PhysicalSetOperation union = Mockito.mock(PhysicalSetOperation.class); + Mockito.when(union.children()).thenReturn(ImmutableList.of(scan1, scan2)); + Mockito.when(union.getRegularChildrenOutputs()) + .thenReturn(ImmutableList.of(ImmutableList.of(k1Left), ImmutableList.of(k1Right))); + Mockito.when(union.getOutput()).thenReturn(ImmutableList.of(setOutputSlot)); + + // A filter kept above the set operation. + Expression filterConjunct = Mockito.mock(Expression.class); + Mockito.when(filterConjunct.getInputSlots()).thenReturn(ImmutableSet.of(setOutputSlot)); + + PhysicalFilter filter = Mockito.mock(PhysicalFilter.class); + Mockito.when(filter.children()).thenReturn(ImmutableList.of(union)); + Mockito.when(filter.getConjuncts()).thenReturn(ImmutableSet.of(filterConjunct)); + Mockito.when(filter.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) filter); + + Assertions.assertEquals(2, deltas.size()); + for (StatsDelta delta : deltas.values()) { + Assertions.assertNotNull(delta.getColumnStats().get("k1")); + Assertions.assertTrue(delta.getColumnStats().get("k1").filterHit, + "k1.filterHit must be true via a filter kept above the set operation"); + } + } + + /** + * SELECT * FROM (SELECT k1 FROM t1 EXCEPT SELECT k1 FROM t2) v WHERE k1 > 0: only t1's + * values ever reach the EXCEPT output (t2 is scanned only to test exclusion), so a filter + * kept above the EXCEPT must resolve to t1's column only, not t2's. + */ + @Test + public void testFilterAboveExceptOnlyResolvesToFirstBranch() { + ExprId leftId = new ExprId(1); + ExprId rightId = new ExprId(3); + ExprId setOutputId = new ExprId(50); + + SlotReference k1Left = mockSlot(leftId, "k1"); + SlotReference k1Right = mockSlot(rightId, "k1"); + PhysicalOlapScan scan1 = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Left)); + PhysicalOlapScan scan2 = mockScan(1L, 2L, 2L, 1L, ImmutableList.of(k1Right)); + + SlotReference setOutputSlot = mockSlot(setOutputId, "k1"); + + PhysicalExcept except = Mockito.mock(PhysicalExcept.class); + Mockito.when(except.children()).thenReturn(ImmutableList.of(scan1, scan2)); + Mockito.when(except.getRegularChildrenOutputs()) + .thenReturn(ImmutableList.of(ImmutableList.of(k1Left), ImmutableList.of(k1Right))); + Mockito.when(except.getOutput()).thenReturn(ImmutableList.of(setOutputSlot)); + + Expression filterConjunct = Mockito.mock(Expression.class); + Mockito.when(filterConjunct.getInputSlots()).thenReturn(ImmutableSet.of(setOutputSlot)); + + PhysicalFilter filter = Mockito.mock(PhysicalFilter.class); + Mockito.when(filter.children()).thenReturn(ImmutableList.of(except)); + Mockito.when(filter.getConjuncts()).thenReturn(ImmutableSet.of(filterConjunct)); + Mockito.when(filter.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) filter); + + // Both branches still get queryHit (they're both genuinely scanned/compared), but only + // the first branch's column gets filterHit from the filter kept above the EXCEPT. + Assertions.assertEquals(2, deltas.size()); + StatsDelta leftDelta = deltas.get("1_1_1_1"); + StatsDelta rightDelta = deltas.get("1_2_2_1"); + Assertions.assertTrue(leftDelta.getColumnStats().get("k1").queryHit); + Assertions.assertTrue(leftDelta.getColumnStats().get("k1").filterHit, + "k1.filterHit must be true on the first (kept) EXCEPT branch"); + Assertions.assertTrue(rightDelta.getColumnStats().get("k1").queryHit); + Assertions.assertFalse(rightDelta.getColumnStats().get("k1").filterHit, + "k1.filterHit must NOT be attributed to the second (subtracted-away) EXCEPT branch"); + } + + /** + * SELECT * FROM (SELECT k1 FROM t1 INTERSECT SELECT k1 FROM t2) v WHERE k1 > 0: like EXCEPT, + * INTERSECT's output block is materialized only from the build-side (first) branch at + * execution time (SetSourceOperatorX::_add_result_columns reads only build_block; probe + * branches only flip a visited bit) — so a filter kept above the INTERSECT must resolve to + * the first branch's column only, not the second. + */ + @Test + public void testFilterAboveIntersectOnlyResolvesToFirstBranch() { + ExprId leftId = new ExprId(1); + ExprId rightId = new ExprId(3); + ExprId setOutputId = new ExprId(50); + + SlotReference k1Left = mockSlot(leftId, "k1"); + SlotReference k1Right = mockSlot(rightId, "k1"); + PhysicalOlapScan scan1 = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Left)); + PhysicalOlapScan scan2 = mockScan(1L, 2L, 2L, 1L, ImmutableList.of(k1Right)); + + SlotReference setOutputSlot = mockSlot(setOutputId, "k1"); + + PhysicalIntersect intersect = Mockito.mock(PhysicalIntersect.class); + Mockito.when(intersect.children()).thenReturn(ImmutableList.of(scan1, scan2)); + Mockito.when(intersect.getRegularChildrenOutputs()) + .thenReturn(ImmutableList.of(ImmutableList.of(k1Left), ImmutableList.of(k1Right))); + Mockito.when(intersect.getOutput()).thenReturn(ImmutableList.of(setOutputSlot)); + + Expression filterConjunct = Mockito.mock(Expression.class); + Mockito.when(filterConjunct.getInputSlots()).thenReturn(ImmutableSet.of(setOutputSlot)); + + PhysicalFilter filter = Mockito.mock(PhysicalFilter.class); + Mockito.when(filter.children()).thenReturn(ImmutableList.of(intersect)); + Mockito.when(filter.getConjuncts()).thenReturn(ImmutableSet.of(filterConjunct)); + Mockito.when(filter.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) filter); + + // Both branches still get queryHit (they're both genuinely scanned/compared), but only + // the first (build-side) branch's column gets filterHit from the filter kept above. + Assertions.assertEquals(2, deltas.size()); + StatsDelta leftDelta = deltas.get("1_1_1_1"); + StatsDelta rightDelta = deltas.get("1_2_2_1"); + Assertions.assertTrue(leftDelta.getColumnStats().get("k1").queryHit); + Assertions.assertTrue(leftDelta.getColumnStats().get("k1").filterHit, + "k1.filterHit must be true on the first (build-side) INTERSECT branch"); + Assertions.assertTrue(rightDelta.getColumnStats().get("k1").queryHit); + Assertions.assertFalse(rightDelta.getColumnStats().get("k1").filterHit, + "k1.filterHit must NOT be attributed to the second (probe-side) INTERSECT branch"); + } + + // ── helpers ────────────────────────────────────────────────────────────── + + /** + * Plan: CTEConsumer (consumerSlot#3 → producerSlot#1=k1) → Scan[k1(#1)] + * Expected: k1.queryHit=true via consumer slot. + */ + @Test + public void testCteConsumerMapsToProducerScan() { + ExprId prodId = new ExprId(1); + ExprId consId = new ExprId(3); + SlotReference prodSlot = mockSlot(prodId, "k1"); + SlotReference consSlot = mockSlot(consId, "k1"); + + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(prodSlot)); + + PhysicalCTEConsumer consumer = Mockito.mock(PhysicalCTEConsumer.class); + Mockito.when(consumer.getOutput()).thenReturn(ImmutableList.of(consSlot)); + Mockito.when(consumer.getProducerSlot(consSlot)).thenReturn(prodSlot); + Mockito.when(consumer.children()).thenReturn(ImmutableList.of()); + + // Root plan: scan (producer) visited first, consumer slots then get registered. + PhysicalFilter root = Mockito.mock(PhysicalFilter.class); + Mockito.when(root.children()).thenReturn(ImmutableList.of(scan, consumer)); + Mockito.when(root.getConjuncts()).thenReturn(ImmutableSet.of()); + Mockito.when(root.getOutput()).thenReturn(ImmutableList.of(consSlot)); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) root); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("k1")); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit); + } + + /** + * Plan: UNION of Scan1[k1(#1)] and Scan2[k1(#3)] + * Expected: k1.queryHit=true on both scan tables. + */ + @Test + public void testUnionRecordsQueryHitOnAllBranches() { + ExprId id1 = new ExprId(1); + ExprId id3 = new ExprId(3); + SlotReference k1Left = mockSlot(id1, "k1"); + SlotReference k1Right = mockSlot(id3, "k1"); + + PhysicalOlapScan scan1 = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Left)); + PhysicalOlapScan scan2 = mockScan(1L, 2L, 2L, 1L, ImmutableList.of(k1Right)); + + PhysicalSetOperation union = Mockito.mock(PhysicalSetOperation.class); + Mockito.when(union.children()).thenReturn(ImmutableList.of(scan1, scan2)); + Mockito.when(union.getRegularChildrenOutputs()) + .thenReturn(ImmutableList.of(ImmutableList.of(k1Left), ImmutableList.of(k1Right))); + Mockito.when(union.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) union); + + Assertions.assertEquals(2, deltas.size()); + for (StatsDelta delta : deltas.values()) { + Assertions.assertNotNull(delta.getColumnStats().get("k1")); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit); + } + } + + /** + * Plan: Filter(SUM(k2)#5 > 0) → Agg[SUM(k2)] → Scan[k2(#2)] + * Expected: k2.filterHit=true from HAVING SUM(k2) > 0. + */ + @Test + @SuppressWarnings("unchecked") + public void testHavingAggregateFilterHitRecorded() { + ExprId k2Id = new ExprId(2); + ExprId sumId = new ExprId(5); + SlotReference k2Slot = mockSlot(k2Id, "k2"); + // Aggregate output slots have no originalColumn in real Doris; use Optional.empty() + // so recordInputSlotsAsFilterHit falls back to exprIdToColName.get(sumId) = "k2". + SlotReference sumSlot = Mockito.mock(SlotReference.class); + Mockito.when(sumSlot.getExprId()).thenReturn(sumId); + Mockito.when(sumSlot.getOriginalColumn()).thenReturn(Optional.empty()); + + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k2Slot)); + + NamedExpression sumExpr = Mockito.mock(NamedExpression.class); + Mockito.when(sumExpr.getExprId()).thenReturn(sumId); + Mockito.when(sumExpr.getInputSlots()).thenReturn(ImmutableSet.of(k2Slot)); + + Aggregate agg = Mockito.mock(Aggregate.class); + Mockito.when(agg.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(agg.getGroupByExpressions()).thenReturn(ImmutableList.of()); + Mockito.when(agg.getOutputExpressions()) + .thenReturn((List) (List) ImmutableList.of(sumExpr)); + Mockito.when(agg.getOutput()).thenReturn(ImmutableList.of(sumSlot)); + + Expression havingConjunct = Mockito.mock(Expression.class); + Mockito.when(havingConjunct.getInputSlots()).thenReturn(ImmutableSet.of(sumSlot)); + + PhysicalFilter havingFilter = Mockito.mock(PhysicalFilter.class); + Mockito.when(havingFilter.children()).thenReturn(ImmutableList.of(agg)); + Mockito.when(havingFilter.getConjuncts()).thenReturn(ImmutableSet.of(havingConjunct)); + Mockito.when(havingFilter.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = + QueryStatsRecorder.collectDeltas((PhysicalPlan) havingFilter); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("k2")); + Assertions.assertTrue(delta.getColumnStats().get("k2").filterHit); + } + + /** + * Plan: Generate(EXPLODE(tags#2)) → Scan[tags(#2)] + * Expected: tags.queryHit=true from the generator input column. + */ + @Test + public void testLateralViewExplodeRecordsGeneratorInputAsQueryHit() { + ExprId tagsId = new ExprId(2); + SlotReference tagsSlot = mockSlot(tagsId, "tags"); + + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(tagsSlot)); + + Function explodeFn = Mockito.mock(Function.class); + Mockito.when(explodeFn.getInputSlots()).thenReturn(ImmutableSet.of(tagsSlot)); + + PhysicalGenerate generate = Mockito.mock(PhysicalGenerate.class); + Mockito.when(generate.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(generate.getGenerators()).thenReturn(ImmutableList.of(explodeFn)); + Mockito.when(generate.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = + QueryStatsRecorder.collectDeltas((PhysicalPlan) generate); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("tags")); + Assertions.assertTrue(delta.getColumnStats().get("tags").queryHit); + Assertions.assertFalse(delta.getColumnStats().get("tags").filterHit); + } + + /** + * SELECT e1 FROM t LATERAL VIEW EXPLODE(arr) tmp AS e1 WHERE e1 > 100: the exploded output + * slot e1 must resolve back to the real scan column (arr) for a filter above the generate. + */ + @Test + @SuppressWarnings("unchecked") + public void testGeneratorOutputSlotResolvesFilterToScanColumn() { + ExprId arrId = new ExprId(2); + ExprId e1Id = new ExprId(3); + SlotReference arrSlot = mockSlot(arrId, "arr"); + SlotReference e1Slot = mockSlot(e1Id, "e1"); + + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(arrSlot)); + + Function explodeFn = Mockito.mock(Function.class); + Mockito.when(explodeFn.getInputSlots()).thenReturn(ImmutableSet.of(arrSlot)); + + PhysicalGenerate generate = Mockito.mock(PhysicalGenerate.class); + Mockito.when(generate.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(generate.getGenerators()).thenReturn(ImmutableList.of(explodeFn)); + Mockito.when(generate.getGeneratorOutput()).thenReturn(ImmutableList.of(e1Slot)); + Mockito.when(generate.getConjuncts()).thenReturn(ImmutableList.of()); + Mockito.when(generate.getOutput()).thenReturn(ImmutableList.of(e1Slot)); + + Expression filterConjunct = Mockito.mock(Expression.class); + Mockito.when(filterConjunct.getInputSlots()).thenReturn(ImmutableSet.of(e1Slot)); + + PhysicalFilter filter = Mockito.mock(PhysicalFilter.class); + Mockito.when(filter.children()).thenReturn(ImmutableList.of(generate)); + Mockito.when(filter.getConjuncts()).thenReturn(ImmutableSet.of(filterConjunct)); + Mockito.when(filter.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) filter); + + StatsDelta delta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(delta); + Assertions.assertTrue(delta.getColumnStats().get("arr").filterHit, + "arr: filter on the exploded output slot resolves back to the real scan column"); + } + + /** + * Plan: PhysicalCTEProducer(Scan[k1(#1)]) + CTEConsumer(consSlot#3 → prodSlot#1) + * PhysicalCTEProducer must walk its child so the scan is registered before the + * consumer is processed by the sibling walk. + * Expected: k1.queryHit=true via consumer slot resolved through producer scan. + */ + @Test + @SuppressWarnings("unchecked") + public void testCTEProducerRegistersScansForConsumer() { + ExprId prodId = new ExprId(1); + ExprId consId = new ExprId(3); + SlotReference prodSlot = mockSlot(prodId, "k1"); + SlotReference consSlot = mockSlot(consId, "k1"); + + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(prodSlot)); + + PhysicalCTEProducer producer = Mockito.mock(PhysicalCTEProducer.class); + Mockito.when(producer.children()).thenReturn(ImmutableList.of(scan)); + + PhysicalCTEConsumer consumer = Mockito.mock(PhysicalCTEConsumer.class); + Mockito.when(consumer.getOutput()).thenReturn(ImmutableList.of(consSlot)); + Mockito.when(consumer.getProducerSlot(consSlot)).thenReturn(prodSlot); + Mockito.when(consumer.children()).thenReturn(ImmutableList.of()); + + PhysicalPlan root = Mockito.mock(PhysicalPlan.class); + Mockito.when(root.children()).thenReturn(ImmutableList.of(producer, consumer)); + Mockito.when(root.getOutput()).thenReturn(ImmutableList.of(consSlot)); + + Map deltas = QueryStatsRecorder.collectDeltas(root); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("k1"), + "k1 must be registered via PhysicalCTEProducer→scan path"); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit, + "k1.queryHit must be true via CTE producer→scan→consumer chain"); + } + + /** + * Plan: Filter(SUM(k2+k3)#5 > 0) → Agg[SUM(k2+k3)] → Scan[k2(#2), k3(#3)] + * Multi-input aggregate output is stored in derivedSlotInputs so that a HAVING + * filter on SUM(k2+k3) records filterHit on both k2 and k3. + * Expected: k2.filterHit=true AND k3.filterHit=true. + */ + @Test + @SuppressWarnings("unchecked") + public void testHavingMultiInputAggregateFilterHitRecorded() { + ExprId k2Id = new ExprId(2); + ExprId k3Id = new ExprId(3); + ExprId sumId = new ExprId(5); + SlotReference k2Slot = mockSlot(k2Id, "k2"); + SlotReference k3Slot = mockSlot(k3Id, "k3"); + SlotReference sumSlot = mockSlot(sumId, "sum_k2_k3"); + + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k2Slot, k3Slot)); + + // SUM(k2+k3): two input slots → must go into derivedSlotInputs + NamedExpression sumExpr = Mockito.mock(NamedExpression.class); + Mockito.when(sumExpr.getExprId()).thenReturn(sumId); + Mockito.when(sumExpr.getInputSlots()).thenReturn(ImmutableSet.of(k2Slot, k3Slot)); + + Aggregate agg = Mockito.mock(Aggregate.class); + Mockito.when(agg.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(agg.getGroupByExpressions()).thenReturn(ImmutableList.of()); + Mockito.when(agg.getOutputExpressions()) + .thenReturn((List) (List) ImmutableList.of(sumExpr)); + Mockito.when(agg.getOutput()).thenReturn(ImmutableList.of(sumSlot)); + + // HAVING SUM(k2+k3) > 0: conjunct references the aggregate output slot + Expression havingConjunct = Mockito.mock(Expression.class); + Mockito.when(havingConjunct.getInputSlots()).thenReturn(ImmutableSet.of(sumSlot)); + + PhysicalFilter havingFilter = Mockito.mock(PhysicalFilter.class); + Mockito.when(havingFilter.children()).thenReturn(ImmutableList.of(agg)); + Mockito.when(havingFilter.getConjuncts()).thenReturn(ImmutableSet.of(havingConjunct)); + Mockito.when(havingFilter.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = + QueryStatsRecorder.collectDeltas((PhysicalPlan) havingFilter); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("k2"), + "k2 must be recorded via multi-input HAVING expansion"); + Assertions.assertTrue(delta.getColumnStats().get("k2").filterHit, + "k2.filterHit must be true from HAVING SUM(k2+k3)"); + Assertions.assertNotNull(delta.getColumnStats().get("k3"), + "k3 must be recorded via multi-input HAVING expansion"); + Assertions.assertTrue(delta.getColumnStats().get("k3").filterHit, + "k3.filterHit must be true from HAVING SUM(k2+k3)"); + } + + /** + * Mark join conjuncts (from IN/EXISTS subquery rewriting) are stored separately + * in AbstractPhysicalJoin and must be processed independently of hash/other conjuncts. + * Plan: HashJoin with k1=k2 in hashJoinConjuncts and k3>0 in markJoinConjuncts. + * Expected: k1.filterHit, k2.filterHit (hash), k3.filterHit (mark). + */ + @Test + @SuppressWarnings("unchecked") + public void testMarkJoinConjunctsRecordFilterHit() { + ExprId id1 = new ExprId(1); + ExprId id2 = new ExprId(2); + ExprId id3 = new ExprId(3); + SlotReference k1Slot = mockSlot(id1, "k1"); + SlotReference k2Slot = mockSlot(id2, "k2"); + SlotReference k3Slot = mockSlot(id3, "k3"); + + PhysicalOlapScan left = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Slot, k3Slot)); + PhysicalOlapScan right = mockScan(2L, 2L, 2L, 2L, ImmutableList.of(k2Slot)); + + Expression hashConjunct = Mockito.mock(Expression.class); + Mockito.when(hashConjunct.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot, k2Slot)); + + // Mark conjunct: k3 > 0 — only reachable via getMarkJoinConjuncts() + Expression markConjunct = Mockito.mock(Expression.class); + Mockito.when(markConjunct.getInputSlots()).thenReturn(ImmutableSet.of(k3Slot)); + + org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin join = + Mockito.mock(org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin.class); + Mockito.when(join.children()).thenReturn(ImmutableList.of(left, right)); + Mockito.when(join.getHashJoinConjuncts()).thenReturn(ImmutableList.of(hashConjunct)); + Mockito.when(join.getOtherJoinConjuncts()).thenReturn(ImmutableList.of()); + Mockito.when(join.getMarkJoinConjuncts()).thenReturn(ImmutableList.of(markConjunct)); + Mockito.when(join.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) join); + + Assertions.assertTrue(deltas.containsKey("1_1_1_1"), "left scan delta missing"); + Assertions.assertTrue(deltas.containsKey("2_2_2_2"), "right scan delta missing"); + Assertions.assertTrue(deltas.get("1_1_1_1").getColumnStats().get("k1").filterHit, + "k1.filterHit: hash join conjunct"); + Assertions.assertTrue(deltas.get("2_2_2_2").getColumnStats().get("k2").filterHit, + "k2.filterHit: hash join conjunct"); + Assertions.assertTrue(deltas.get("1_1_1_1").getColumnStats().get("k3").filterHit, + "k3.filterHit: mark join conjunct — previously missed"); + } + + /** + * Other join conjuncts (non-equi predicates, e.g. a range condition in a nested-loop join) + * are stored separately from hash/mark conjuncts and must be processed too. + * Plan: NestedLoopJoin with k1=k2 in hashJoinConjuncts and k3 join = + Mockito.mock(org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin.class); + Mockito.when(join.children()).thenReturn(ImmutableList.of(left, right)); + Mockito.when(join.getHashJoinConjuncts()).thenReturn(ImmutableList.of(hashConjunct)); + Mockito.when(join.getOtherJoinConjuncts()).thenReturn(ImmutableList.of(otherConjunct)); + Mockito.when(join.getMarkJoinConjuncts()).thenReturn(ImmutableList.of()); + Mockito.when(join.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) join); + + Assertions.assertTrue(deltas.containsKey("1_1_1_1"), "left scan delta missing"); + Assertions.assertTrue(deltas.containsKey("2_2_2_2"), "right scan delta missing"); + Assertions.assertTrue(deltas.get("1_1_1_1").getColumnStats().get("k1").filterHit, + "k1.filterHit: hash join conjunct"); + Assertions.assertTrue(deltas.get("2_2_2_2").getColumnStats().get("k2").filterHit, + "k2.filterHit: hash join conjunct"); + Assertions.assertTrue(deltas.get("1_1_1_1").getColumnStats().get("k3").filterHit, + "k3.filterHit: other join conjunct — previously untested"); + Assertions.assertTrue(deltas.get("2_2_2_2").getColumnStats().get("k4").filterHit, + "k4.filterHit: other join conjunct — previously untested"); + } + + /** + * PhysicalRecursiveUnion extends PhysicalBinary: left() is the base case, right() is the + * recursive case. The base-case slots (from an OlapScan) must get queryHit. + * Expected: k1.queryHit=true (base case only, exactly one scan delta). + */ + @Test + @SuppressWarnings("unchecked") + public void testRecursiveUnionBaseColumnGetsQueryHit() { + ExprId baseScanId = new ExprId(1); + ExprId recursiveId = new ExprId(5); // WorkTableReference slot, no CTEId link established + SlotReference baseScanSlot = mockSlot(baseScanId, "k1"); + SlotReference recursiveSlot = mockSlot(recursiveId, "k1"); + + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(baseScanSlot)); + + // Mock the recursive child (not a PhysicalWorkTableReference, no CTEId link is set up). + Plan recursiveChild = Mockito.mock(Plan.class); + Mockito.when(recursiveChild.children()).thenReturn(ImmutableList.of()); + Mockito.when(recursiveChild.getOutput()).thenReturn(ImmutableList.of(recursiveSlot)); + + PhysicalRecursiveUnion recUnion = Mockito.mock(PhysicalRecursiveUnion.class); + Mockito.when(recUnion.left()).thenReturn((Plan) scan); + Mockito.when(recUnion.right()).thenReturn(recursiveChild); + Mockito.when(recUnion.getRegularChildrenOutputs()).thenReturn( + ImmutableList.of( + ImmutableList.of(baseScanSlot), // base case: resolves to scan + ImmutableList.of(recursiveSlot) // recursive case: no work-table CTEId link + )); + Mockito.when(recUnion.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) recUnion); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(delta, "base-case scan delta must exist"); + Assertions.assertNotNull(delta.getColumnStats().get("k1"), + "k1 must be recorded from base case via getRegularChildrenOutputs"); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit, + "k1.queryHit must be true for recursive union base case"); + // Exactly one delta entry — recursive-case slot (#5) must not create a second scan entry. + Assertions.assertEquals(1, deltas.size(), "only base-case scan must appear in deltas"); + } + + /** + * SELECT * FROM t1 WHERE k1 = 1 UNION ALL SELECT * FROM cte WHERE k1 = 2 (recursive branch, + * filtering on the work-table's own re-scanned column). The base case is wrapped in a + * PhysicalRecursiveUnionAnchor exposing a CTEId; the recursive case contains a filter over a + * PhysicalWorkTableReference with that same CTEId. Before this fix, the work-table slot's + * ExprId had no entry in derivedSlotInputs by the time the recursive branch's own filter was + * walked, so k1.filterHit could never be recorded for the recursive-side predicate. + * Expected: k1.filterHit=true (recursive-side WHERE resolves back to the real scan column). + */ + @Test + @SuppressWarnings("unchecked") + public void testFilterOnWorkTableSlotInRecursiveBranchRecordsFilterHit() { + CTEId cteId = new CTEId(1); + ExprId baseScanId = new ExprId(1); + ExprId workTableId = new ExprId(5); + SlotReference baseScanSlot = mockSlot(baseScanId, "k1"); + SlotReference workTableSlot = mockSlot(workTableId, "k1"); + + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(baseScanSlot)); + + PhysicalRecursiveUnionAnchor anchor = Mockito.mock(PhysicalRecursiveUnionAnchor.class); + Mockito.when(anchor.getCteId()).thenReturn(cteId); + Mockito.when(anchor.children()).thenReturn(ImmutableList.of(scan)); + + PhysicalWorkTableReference workTable = Mockito.mock(PhysicalWorkTableReference.class, + Mockito.CALLS_REAL_METHODS); + Mockito.when(workTable.getCteId()).thenReturn(cteId); + Mockito.when(workTable.getOutput()).thenReturn(ImmutableList.of(workTableSlot)); + Mockito.when(workTable.children()).thenReturn(ImmutableList.of()); + + // WHERE k1 = 2 inside the recursive branch, referencing the work-table's own slot. + Expression recursiveFilterExpr = Mockito.mock(Expression.class); + Mockito.when(recursiveFilterExpr.getInputSlots()).thenReturn(ImmutableSet.of(workTableSlot)); + + // CALLS_REAL_METHODS so collectToList()/foreach() (default TreeNode methods used by the + // production code to locate the work-table reference) actually walk into the stubbed + // children() below, instead of Mockito's default answer short-circuiting to an empty result. + PhysicalFilter recursiveFilter = Mockito.mock(PhysicalFilter.class, Mockito.CALLS_REAL_METHODS); + Mockito.when(recursiveFilter.children()).thenReturn(ImmutableList.of(workTable)); + Mockito.when(recursiveFilter.getConjuncts()).thenReturn(ImmutableSet.of(recursiveFilterExpr)); + + PhysicalRecursiveUnion recUnion = Mockito.mock(PhysicalRecursiveUnion.class); + Mockito.when(recUnion.left()).thenReturn((Plan) anchor); + Mockito.when(recUnion.right()).thenReturn(recursiveFilter); + Mockito.when(recUnion.getRegularChildrenOutputs()).thenReturn( + ImmutableList.of( + ImmutableList.of(baseScanSlot), + ImmutableList.of(workTableSlot))); + Mockito.when(recUnion.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) recUnion); + + StatsDelta delta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(delta, "base scan delta must exist"); + Assertions.assertTrue(delta.getColumnStats().get("k1").filterHit, + "recursive-branch filter on work-table slot must resolve back to the real scan column"); + } + + /** + * SELECT k1+k2 AS result FROM t: computed alias with two input slots. + * unwrapAlias returns null (not a plain SlotReference), so the PhysicalProject handler + * stores {k1, k2} in the expansion map (derivedSlotInputs). The root output loop in + * collectDeltas then expands via that map and records k1.queryHit and k2.queryHit. + * Expected: k1.queryHit=true AND k2.queryHit=true. + */ + @Test + @SuppressWarnings("unchecked") + public void testComputedProjectExpressionRecordsBothInputsAsQueryHit() { + ExprId id1 = new ExprId(1); + ExprId id2 = new ExprId(2); + ExprId aliasId = new ExprId(99); + SlotReference k1Slot = mockSlot(id1, "k1"); + SlotReference k2Slot = mockSlot(id2, "k2"); + PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Slot, k2Slot)); + + // Simulates k1+k2 — a binary expression with two input slots + Expression addExpr = Mockito.mock(Expression.class); + Mockito.when(addExpr.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot, k2Slot)); + + org.apache.doris.nereids.trees.expressions.Alias alias = + Mockito.mock(org.apache.doris.nereids.trees.expressions.Alias.class); + Mockito.when(alias.getExprId()).thenReturn(aliasId); + Mockito.when(alias.child()).thenReturn(addExpr); + + org.apache.doris.nereids.trees.plans.physical.PhysicalProject project = + Mockito.mock(org.apache.doris.nereids.trees.plans.physical.PhysicalProject.class); + Mockito.when(project.children()).thenReturn(ImmutableList.of(scan)); + Mockito.when(project.getProjects()).thenReturn(ImmutableList.of(alias)); + Mockito.when(project.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas( + (org.apache.doris.nereids.trees.plans.physical.PhysicalPlan) project); + + Assertions.assertEquals(1, deltas.size()); + StatsDelta delta = deltas.values().iterator().next(); + Assertions.assertNotNull(delta.getColumnStats().get("k1"), + "k1 must be recorded as queryHit from computed SELECT k1+k2"); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit, + "k1.queryHit must be true"); + Assertions.assertNotNull(delta.getColumnStats().get("k2"), + "k2 must be recorded as queryHit from computed SELECT k1+k2"); + Assertions.assertTrue(delta.getColumnStats().get("k2").queryHit, + "k2.queryHit must be true"); + } + + /** + * PushDownExpressionsInHashCondition inserts a PhysicalProject below the join with + * Alias(k1+k2) as the hash-join key. Walking this intermediate project must NOT record + * k1/k2 as queryHit; instead, the join hash conjunct (which references the alias slot) + * must expand via the map and record k1.filterHit and k2.filterHit. + * Plan: HashJoin(aliasSlot = k3) → Project([Alias(k1+k2)]) → Scan[k1,k2]; Scan[k3] + * Expected: k1.filterHit=true, k2.filterHit=true, k1.queryHit=false, k2.queryHit=false. + */ + @Test + @SuppressWarnings("unchecked") + public void testIntermediateProjectComputedJoinKeyRecordsFilterHitNotQueryHit() { + ExprId id1 = new ExprId(1); + ExprId id2 = new ExprId(2); + ExprId id3 = new ExprId(3); + ExprId aliasId = new ExprId(99); + SlotReference k1Slot = mockSlot(id1, "k1"); + SlotReference k2Slot = mockSlot(id2, "k2"); + SlotReference k3Slot = mockSlot(id3, "k3"); + PhysicalOlapScan leftScan = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Slot, k2Slot)); + PhysicalOlapScan rightScan = mockScan(2L, 2L, 2L, 2L, ImmutableList.of(k3Slot)); + + // Simulates k1+k2 — a binary expression with two input slots + Expression addExpr = Mockito.mock(Expression.class); + Mockito.when(addExpr.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot, k2Slot)); + + org.apache.doris.nereids.trees.expressions.Alias alias = + Mockito.mock(org.apache.doris.nereids.trees.expressions.Alias.class); + Mockito.when(alias.getExprId()).thenReturn(aliasId); + Mockito.when(alias.child()).thenReturn(addExpr); + + // The slot the join conjunct references (the alias output, not k1/k2 directly) + SlotReference aliasSlot = mockSlot(aliasId, "k1_plus_k2"); + + org.apache.doris.nereids.trees.plans.physical.PhysicalProject project = + Mockito.mock(org.apache.doris.nereids.trees.plans.physical.PhysicalProject.class); + Mockito.when(project.children()).thenReturn(ImmutableList.of(leftScan)); + Mockito.when(project.getProjects()).thenReturn(ImmutableList.of(alias)); + Mockito.when(project.getOutput()).thenReturn(ImmutableList.of(aliasSlot)); + + // Hash conjunct: (k1+k2) = k3 — references aliasSlot, not k1/k2 directly + Expression hashConjunct = Mockito.mock(Expression.class); + Mockito.when(hashConjunct.getInputSlots()).thenReturn(ImmutableSet.of(aliasSlot, k3Slot)); + + org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin join = + Mockito.mock(org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin.class); + Mockito.when(join.children()).thenReturn(ImmutableList.of(project, rightScan)); + Mockito.when(join.getHashJoinConjuncts()).thenReturn(ImmutableList.of(hashConjunct)); + Mockito.when(join.getOtherJoinConjuncts()).thenReturn(ImmutableList.of()); + Mockito.when(join.getOutput()).thenReturn(ImmutableList.of()); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) join); + + Assertions.assertTrue(deltas.containsKey("1_1_1_1"), "left scan delta must exist"); + StatsDelta leftDelta = deltas.get("1_1_1_1"); + Assertions.assertNotNull(leftDelta.getColumnStats().get("k1"), + "k1 must be recorded via join-key alias expansion"); + Assertions.assertTrue(leftDelta.getColumnStats().get("k1").filterHit, + "k1.filterHit must be true: used only in join predicate"); + Assertions.assertFalse(leftDelta.getColumnStats().get("k1").queryHit, + "k1.queryHit must be false: intermediate project must not record queryHit"); + Assertions.assertNotNull(leftDelta.getColumnStats().get("k2"), + "k2 must be recorded via join-key alias expansion"); + Assertions.assertTrue(leftDelta.getColumnStats().get("k2").filterHit, + "k2.filterHit must be true: used only in join predicate"); + Assertions.assertFalse(leftDelta.getColumnStats().get("k2").queryHit, + "k2.queryHit must be false: intermediate project must not record queryHit"); + } + + /** + * SELECT u.k1 FROM (SELECT k1, k2 FROM t1 EXCEPT SELECT k1, k2 FROM t2) u + * The outer PhysicalProject only selects k1; k2 is never read by the visible query, but it + * is still physically scanned and compared by the EXCEPT itself (ColumnPruning intentionally + * retains it for row comparison — see the "Accepted as-is" note in the PR description for #6). + * Expected: k2 still gets queryHit=true on BOTH branches, since both branches' k2 columns are + * genuinely read to perform the row comparison, even though neither is in the final SELECT list. + */ + @Test + public void adversarialExceptUnusedColumnStillGetsQueryHitForRowComparison() { + ExprId k1LeftId = new ExprId(1); + ExprId k2LeftId = new ExprId(2); + ExprId k1RightId = new ExprId(3); + ExprId k2RightId = new ExprId(4); + + SlotReference k1Left = mockSlot(k1LeftId, "k1"); + SlotReference k2Left = mockSlot(k2LeftId, "k2"); + SlotReference k1Right = mockSlot(k1RightId, "k1"); + SlotReference k2Right = mockSlot(k2RightId, "k2"); + + PhysicalOlapScan scan1 = mockScan(1L, 1L, 1L, 1L, ImmutableList.of(k1Left, k2Left)); + PhysicalOlapScan scan2 = mockScan(1L, 2L, 2L, 1L, ImmutableList.of(k1Right, k2Right)); + + ExprId setK1Id = new ExprId(10); + ExprId setK2Id = new ExprId(11); + SlotReference setK1 = mockSlot(setK1Id, "k1"); + SlotReference setK2 = mockSlot(setK2Id, "k2"); + + PhysicalExcept except = Mockito.mock(PhysicalExcept.class); + Mockito.when(except.children()).thenReturn(ImmutableList.of(scan1, scan2)); + Mockito.when(except.getRegularChildrenOutputs()) + .thenReturn(ImmutableList.of( + ImmutableList.of(k1Left, k2Left), + ImmutableList.of(k1Right, k2Right))); + Mockito.when(except.getOutput()).thenReturn(ImmutableList.of(setK1, setK2)); + + // Outer PhysicalProject: SELECT u.k1 -- only k1 is a pass-through, k2 is dropped. + PhysicalProject outerProject = Mockito.mock(PhysicalProject.class); + Mockito.when(outerProject.children()).thenReturn(ImmutableList.of(except)); + Mockito.when(outerProject.getProjects()).thenReturn(ImmutableList.of(setK1)); + Mockito.when(outerProject.getOutput()).thenReturn(ImmutableList.of(setK1)); + + Map deltas = QueryStatsRecorder.collectDeltas((PhysicalPlan) outerProject); + + Assertions.assertEquals(2, deltas.size()); + for (StatsDelta delta : deltas.values()) { + Assertions.assertNotNull(delta.getColumnStats().get("k1")); + Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit, + "k1 is selected by the outer query and must get queryHit"); + if (delta.getColumnStats().containsKey("k2")) { + Assertions.assertTrue(delta.getColumnStats().get("k2").queryHit, + "k2 is retained by ColumnPruning for EXCEPT row comparison, so it is " + + "genuinely read and must get queryHit even though it's not in " + + "the outer SELECT list — this is the accepted #6 behavior"); + } + } + } private SlotReference mockSlot(ExprId exprId, String columnName) { SlotReference slot = Mockito.mock(SlotReference.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index 8bccbd2929f665..af9bd9d2ac378f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -44,21 +44,34 @@ import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergHadoopExternalCatalog; +import org.apache.doris.datasource.iceberg.helper.IcebergWriterHelper; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.qe.SessionVariable; import org.apache.doris.rpc.RpcException; import org.apache.doris.statistics.AnalysisManager; import org.apache.doris.statistics.ColStatsMeta; import org.apache.doris.statistics.TableStatsMeta; +import org.apache.doris.thrift.TIcebergColumnStats; +import org.apache.doris.thrift.TIcebergCommitData; import org.apache.doris.thrift.TStorageType; import com.google.common.collect.Maps; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.io.IOException; import java.time.LocalTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -67,6 +80,63 @@ import java.util.Map; class StatisticsUtilTest { + @Test + void testGetIcebergColumnStatsReturnsEmptyForDisabledMetrics() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).build(); + org.apache.iceberg.Table table = Mockito.mock(org.apache.iceberg.Table.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(table.spec()).thenReturn(spec); + Mockito.when(table.sortOrder()).thenReturn(SortOrder.unsorted()); + Mockito.when(table.properties()).thenReturn(Map.of( + TableProperties.DEFAULT_FILE_FORMAT, "parquet", + TableProperties.DEFAULT_WRITE_METRICS_MODE, "none")); + + TIcebergColumnStats columnStats = new TIcebergColumnStats(); + columnStats.setColumnSizes(Map.of(1, 128L)); + columnStats.setValueCounts(Map.of(1, 10L)); + columnStats.setNullValueCounts(Map.of(1, 0L)); + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("/path/to/data.parquet"); + commitData.setRowCount(10); + commitData.setFileSize(1024); + commitData.setColumnStats(columnStats); + DataFile dataFile = IcebergWriterHelper.convertToWriterResult(table, List.of(commitData)).dataFiles()[0]; + + TableScan tableScan = Mockito.mock(TableScan.class); + FileScanTask fileScanTask = Mockito.mock(FileScanTask.class); + Mockito.when(table.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan); + Mockito.when(tableScan.planFiles()) + .thenReturn(CloseableIterable.withNoopClose(List.of(fileScanTask))); + Mockito.when(fileScanTask.spec()).thenReturn(spec); + Mockito.when(fileScanTask.file()).thenReturn(dataFile); + + Assertions.assertTrue(StatisticsUtil.getIcebergColumnStats("id", table).isEmpty()); + } + + @Test + void testGetIcebergColumnStatsReturnsEmptyWhenCloseFails() { + Schema schema = new Schema(Types.NestedField.optional(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).build(); + org.apache.iceberg.Table table = Mockito.mock(org.apache.iceberg.Table.class); + TableScan tableScan = Mockito.mock(TableScan.class); + FileScanTask fileScanTask = Mockito.mock(FileScanTask.class); + DataFile dataFile = Mockito.mock(DataFile.class); + Mockito.when(table.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.includeColumnStats()).thenReturn(tableScan); + Mockito.when(tableScan.planFiles()).thenReturn(CloseableIterable.combine( + List.of(fileScanTask), () -> { + throw new IOException("close failed"); + })); + Mockito.when(fileScanTask.spec()).thenReturn(spec); + Mockito.when(fileScanTask.file()).thenReturn(dataFile); + Mockito.when(dataFile.columnSizes()).thenReturn(Map.of()); + Mockito.when(dataFile.nullValueCounts()).thenReturn(Map.of()); + + Assertions.assertTrue(StatisticsUtil.getIcebergColumnStats("id", table).isEmpty()); + } + @Test void testConvertToDouble() { try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/CdcStreamTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/CdcStreamTableValuedFunctionTest.java index 1be26fa69daa7f..09eb838f759353 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/CdcStreamTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/CdcStreamTableValuedFunctionTest.java @@ -22,9 +22,11 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.jdbc.client.JdbcClient; import org.apache.doris.job.cdc.DataSourceConfigKeys; +import org.apache.doris.job.cdc.request.FetchRecordRequest; import org.apache.doris.job.common.DataSourceType; import org.apache.doris.job.util.StreamingJobUtils; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Assert; import org.junit.Test; import org.mockito.MockedStatic; @@ -37,6 +39,8 @@ public class CdcStreamTableValuedFunctionTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Test public void testDeleteSignIsExcludedByDefault() throws Exception { List columns = getTableColumns(baseProperties()); @@ -70,6 +74,17 @@ public void testInvalidIncludeDeleteSignIsRejected() { Assert.assertTrue(exception.getMessage().contains("include_delete_sign")); } + @Test + public void testMysqlJdbcUrlIsNormalizedInPayload() throws Exception { + CdcStreamTableValuedFunction function = new CdcStreamTableValuedFunction(baseProperties()); + + FetchRecordRequest request = OBJECT_MAPPER.readValue( + function.getBackendConnectProperties().get("http.payload"), FetchRecordRequest.class); + Assert.assertEquals("jdbc:mysql://localhost:3306/test_db?yearIsDateType=false" + + "&tinyInt1isBit=false&useUnicode=true&characterEncoding=utf-8", + request.getConfig().get(DataSourceConfigKeys.JDBC_URL)); + } + private List getTableColumns(Map properties) throws Exception { JdbcClient jdbcClient = Mockito.mock(JdbcClient.class); List sourceColumns = new ArrayList<>(); diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/S3CompatibleFileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/S3CompatibleFileSystemProperties.java index db59fe42004cc4..10a92728666189 100644 --- a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/S3CompatibleFileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/properties/S3CompatibleFileSystemProperties.java @@ -17,6 +17,8 @@ package org.apache.doris.filesystem.properties; +import java.util.Set; + /** * Shared typed accessors for S3-compatible object storage properties. * @@ -65,6 +67,16 @@ public interface S3CompatibleFileSystemProperties extends FileSystemProperties { /** Returns whether path-style bucket addressing is enabled, as a raw provider property value. */ String getUsePathStyle(); + /** + * Returns the URI schemes this provider accepts, lower-cased (e.g. {@code {s3, s3a, oss}}). + * + *

This is the single source of truth for the provider's scheme identity: URI parsing + * rejects any scheme not in this set (so a COS provider refuses {@code oss://}), and + * scheme-to-storage routing can read the same set. Every S3-compatible provider accepts the + * historically normalized {@code s3}/{@code s3a} form in addition to its native scheme(s). + */ + Set getSupportedSchemes(); + /** * Returns path-style bucket addressing as a parsed boolean (single conversion point). * diff --git a/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/properties/S3CompatibleFileSystemPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/properties/S3CompatibleFileSystemPropertiesTest.java index 83af16095beab3..ac1f6ebeedc5fa 100644 --- a/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/properties/S3CompatibleFileSystemPropertiesTest.java +++ b/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/properties/S3CompatibleFileSystemPropertiesTest.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.Map; +import java.util.Set; public class S3CompatibleFileSystemPropertiesTest { @@ -94,6 +95,11 @@ public String getUsePathStyle() { return usePathStyle; } + @Override + public Set getSupportedSchemes() { + return Set.of("s3", "s3a"); + } + @Override public String providerName() { return "TEST"; diff --git a/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosFileSystem.java b/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosFileSystem.java index e6f206897f8437..3b805953ed61ca 100644 --- a/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosFileSystem.java @@ -25,6 +25,6 @@ public class CosFileSystem extends S3CompatibleFileSystem { public CosFileSystem(CosObjStorage objStorage) { - super(objStorage, objStorage.isUsePathStyle()); + super(objStorage, objStorage.isUsePathStyle(), objStorage.getSupportedSchemes()); } } diff --git a/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosFileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosFileSystemProperties.java index 2d3478169465e9..8a70465c6b3985 100644 --- a/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosFileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosFileSystemProperties.java @@ -36,6 +36,7 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -332,6 +333,11 @@ public String getUsePathStyle() { return usePathStyle; } + @Override + public Set getSupportedSchemes() { + return Set.of("cos", "cosn", "s3", "s3a"); + } + public String getForceParsingByStandardUrl() { return forceParsingByStandardUrl; } diff --git a/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosObjStorage.java b/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosObjStorage.java index 52923efb89267c..09be61b1c31fd4 100644 --- a/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosObjStorage.java @@ -69,6 +69,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -103,6 +104,11 @@ public boolean isUsePathStyle() { return properties.isUsePathStyle(); } + /** Returns the URI schemes this provider accepts (e.g. {@code {cos, cosn, s3, s3a}}). */ + public Set getSupportedSchemes() { + return properties.getSupportedSchemes(); + } + @Override public COSClient getClient() throws IOException { if (closed.get()) { @@ -121,6 +127,15 @@ public COSClient getClient() throws IOException { protected COSClient buildCosClient(String region) throws IOException { COSCredentials cred = buildCredentials(); ClientConfig clientConfig = new ClientConfig(); + // Note on use_path_style: unlike the S3/OSS/OBS SDKs, the native COS SDK has no + // path-style addressing mode — it always renders the bucket as a host subdomain + // (bucket-appid.) and puts only the object key in the request path + // (see COSClient#buildUrlAndHost). There is therefore no client setting to apply here. + // This is harmless for COS: bucket names are always of the form name-appid, which is a + // valid DNS label, so virtual-hosted access always works. The use_path_style flag is + // still honored where it matters — URI parsing in S3CompatibleFileSystem — so a + // path-style URL supplied by the user is parsed into the correct (bucket, key) and the + // request is then re-issued virtual-hosted to that same bucket. clientConfig.setRegion(new Region(region)); clientConfig.setHttpProtocol(HttpProtocol.https); clientConfig.setEndPointSuffix(stripScheme(properties.getEndpoint())); @@ -155,7 +170,7 @@ public RemoteObjects listObjects(String remotePath, String continuationToken) th @Override public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions options) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(uri.bucket()); request.setPrefix(uri.key()); @@ -186,7 +201,7 @@ public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions @Override public RemoteObject headObject(String remotePath) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { ObjectMetadata metadata = getClient().getObjectMetadata(uri.bucket(), uri.key()); return new RemoteObject(uri.key(), uri.key(), metadata.getETag(), metadata.getContentLength(), @@ -203,7 +218,7 @@ public RemoteObject headObject(String remotePath) throws IOException { @Override public void putObject(String remotePath, RequestBody requestBody) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(requestBody.contentLength()); try (InputStream content = requestBody.content()) { @@ -215,7 +230,7 @@ public void putObject(String remotePath, RequestBody requestBody) throws IOExcep @Override public void deleteObject(String remotePath) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { getClient().deleteObject(uri.bucket(), uri.key()); } catch (CosServiceException e) { @@ -230,8 +245,8 @@ public void deleteObject(String remotePath) throws IOException { @Override public void copyObject(String srcPath, String dstPath) throws IOException { - ObjectStorageUri src = ObjectStorageUri.parse(srcPath, false); - ObjectStorageUri dst = ObjectStorageUri.parse(dstPath, false); + ObjectStorageUri src = ObjectStorageUri.parse(srcPath, isUsePathStyle(), getSupportedSchemes()); + ObjectStorageUri dst = ObjectStorageUri.parse(dstPath, isUsePathStyle(), getSupportedSchemes()); try { getClient().copyObject(new CopyObjectRequest( src.bucket(), src.key(), dst.bucket(), dst.key())); @@ -243,7 +258,7 @@ public void copyObject(String srcPath, String dstPath) throws IOException { @Override public String initiateMultipartUpload(String remotePath) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { InitiateMultipartUploadResult result = getClient().initiateMultipartUpload( new InitiateMultipartUploadRequest(uri.bucket(), uri.key())); @@ -257,7 +272,7 @@ public String initiateMultipartUpload(String remotePath) throws IOException { @Override public UploadPartResult uploadPart(String remotePath, String uploadId, int partNum, RequestBody body) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try (InputStream content = body.content()) { com.qcloud.cos.model.UploadPartRequest request = new com.qcloud.cos.model.UploadPartRequest(); request.setBucketName(uri.bucket()); @@ -277,7 +292,7 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN @Override public void completeMultipartUpload(String remotePath, String uploadId, List parts) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); List partEtags = parts.stream() .map(part -> new PartETag(part.partNumber(), part.etag())) .collect(Collectors.toList()); @@ -292,7 +307,7 @@ public void completeMultipartUpload(String remotePath, String uploadId, @Override public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { getClient().abortMultipartUpload(new AbortMultipartUploadRequest( uri.bucket(), uri.key(), uploadId)); @@ -304,7 +319,7 @@ public void abortMultipartUpload(String remotePath, String uploadId) throws IOEx @Override public InputStream openInputStreamAt(String remotePath, long fromByte) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { GetObjectRequest request = new GetObjectRequest(uri.bucket(), uri.key()); if (fromByte > 0) { @@ -382,7 +397,7 @@ public String getPresignedUrl(String objectKey) throws IOException { Date expiration = new Date(System.currentTimeMillis() + (long) SESSION_EXPIRE_SECONDS * 1000); URL url = cos.generatePresignedUrl(bucket, objectKey, expiration, HttpMethodName.PUT, new HashMap<>(), new HashMap<>()); - LOG.info("Generated COS presigned URL for key={}", objectKey); + LOG.debug("Generated COS presigned URL for key={}", objectKey); return url.toString(); } catch (CosClientException e) { LOG.warn("Failed to generate COS presigned URL for key={} in region={}", diff --git a/fe/fe-filesystem/fe-filesystem-hdfs-base/pom.xml b/fe/fe-filesystem/fe-filesystem-hdfs-base/pom.xml new file mode 100644 index 00000000000000..cac69823ff62a3 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-hdfs-base/pom.xml @@ -0,0 +1,111 @@ + + + + 4.0.0 + + + org.apache.doris + fe-filesystem + ${revision} + ../pom.xml + + + fe-filesystem-hdfs-base + jar + Doris FE Filesystem - HDFS base (shared library) + Shared HDFS-compatible code (DFSFileSystem, auth, config, base properties) reused by the + hdfs / oss-hdfs / jfs plugins. Not a plugin itself: no provider, no META-INF/services. + + + + org.apache.doris + fe-filesystem-api + ${revision} + + provided + + + org.apache.doris + fe-filesystem-spi + ${revision} + + provided + + + org.apache.doris + fe-foundation + ${revision} + + + org.projectlombok + lombok + provided + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + org.apache.hadoop + hadoop-client + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + org.apache.hadoop + hadoop-auth + + + org.apache.logging.log4j + log4j-api + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-core + test + + + diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilder.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilder.java similarity index 73% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilder.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilder.java index 8c100f80e31c02..5ae6180907be5b 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilder.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilder.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import java.util.Map; +import java.util.Set; /** * Builds a Hadoop {@link Configuration} from a {@code Map} of properties. @@ -30,20 +31,29 @@ public class HdfsConfigBuilder { static final String KEY_PRINCIPAL = "hadoop.kerberos.principal"; static final String KEY_KEYTAB = "hadoop.kerberos.keytab"; + /** + * Schemes whose Hadoop FS cache must be disabled so {@link DFSFileSystem#close()} only closes + * instances this module owns. This is the union of every scheme served by a DFSFileSystem in + * this module, including {@code oss} for the OSS-HDFS (JindoFS) path — note this is broader + * than {@link HdfsFileSystemProvider#SUPPORTED_SCHEMES}, which is the plain-HDFS routing set + * and intentionally excludes {@code oss} (owned by {@link OssHdfsFileSystemProvider}). + */ + static final Set CACHE_DISABLE_SCHEMES = Set.of("hdfs", "viewfs", "ofs", "jfs", "oss"); + private HdfsConfigBuilder() { } /** * Builds a Hadoop Configuration from the given properties map. - * FS caching is disabled for every scheme the provider claims to support so that + * FS caching is disabled for every scheme this module can serve so that * {@link DFSFileSystem#close()} only closes the FileSystem instances owned by this - * provider — never a globally-cached instance that another catalog is still using. + * module — never a globally-cached instance that another catalog is still using. */ public static Configuration build(Map properties) { Configuration conf = new HdfsConfiguration(); conf.setBoolean("fs.hdfs.impl.disable.cache", true); conf.setBoolean("fs.AbstractFileSystem.hdfs.impl.disable.cache", true); - for (String scheme : HdfsFileSystemProvider.SUPPORTED_SCHEMES) { + for (String scheme : CACHE_DISABLE_SCHEMES) { conf.setBoolean("fs." + scheme + ".impl.disable.cache", true); conf.setBoolean("fs.AbstractFileSystem." + scheme + ".impl.disable.cache", true); } diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileIterator.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileIterator.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileIterator.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileIterator.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsInputFile.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsInputFile.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsInputFile.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsInputFile.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsOutputFile.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsOutputFile.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsOutputFile.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsOutputFile.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsSeekableInputStream.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsSeekableInputStream.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsSeekableInputStream.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/HdfsSeekableInputStream.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/KerberosHadoopAuthenticator.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/KerberosHadoopAuthenticator.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/KerberosHadoopAuthenticator.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/KerberosHadoopAuthenticator.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/SimpleHadoopAuthenticator.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/SimpleHadoopAuthenticator.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/SimpleHadoopAuthenticator.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/SimpleHadoopAuthenticator.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsCompatibleProperties.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsCompatibleProperties.java new file mode 100644 index 00000000000000..c7fd4d1cf0a6a9 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsCompatibleProperties.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs.properties; + +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; +import org.apache.doris.foundation.property.StoragePropertiesException; + +import com.google.common.base.Strings; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +/** + * Shared base for HDFS-compatible storage property models in fe-filesystem. + * + *

Both plain HDFS ({@link HdfsProperties}) and Aliyun OSS-HDFS (JindoFS) bind + * {@code @ConnectorProperty} fields via fe-foundation, then derive a Hadoop-style backend + * configuration map that is handed to {@code DFSFileSystem}. This base owns everything that is + * identical between the two: raw-property retention, {@code @ConnectorProperty} binding, the + * generic required-field check, xml-resource loading, and the backend-config accessor. The + * provider-specific normalization (auth translation for HDFS, Jindo wiring for OSS-HDFS) is + * supplied by subclasses via {@link #doInitNormalizeAndCheckProps()}.

+ * + *

Zero fe-core / fe-common dependency — only fe-foundation.

+ */ +public abstract class HdfsCompatibleProperties { + + public static final String HDFS_DEFAULT_FS_NAME = "fs.defaultFS"; + + protected final Map origProps; + + protected Map backendConfigProperties; + + protected HdfsCompatibleProperties(Map origProps) { + this.origProps = origProps; + } + + /** + * Binds the {@code @ConnectorProperty} fields from the raw properties, validates them, then + * delegates the provider-specific derivation of {@link #backendConfigProperties} to the + * subclass. + */ + public void initNormalizeAndCheckProps() { + ConnectorPropertiesUtils.bindConnectorProperties(this, origProps); + checkRequiredProperties(); + doInitNormalizeAndCheckProps(); + } + + /** + * Generic reflection-based validation of {@code required=true} string fields, shared by all + * HDFS-compatible types. Subclasses may override to add type-specific checks (calling + * {@code super.checkRequiredProperties()} first). + */ + protected void checkRequiredProperties() { + for (Field field : ConnectorPropertiesUtils.getConnectorProperties(this.getClass())) { + field.setAccessible(true); + ConnectorProperty anno = field.getAnnotation(ConnectorProperty.class); + String[] names = anno.names(); + if (anno.required() && field.getType().equals(String.class)) { + try { + String value = (String) field.get(this); + if (Strings.isNullOrEmpty(value)) { + throw new IllegalArgumentException("Property " + names[0] + " is required."); + } + } catch (IllegalAccessException e) { + throw new StoragePropertiesException("Failed to get property " + names[0] + + ", " + e.getMessage(), e); + } + } + } + } + + /** + * Provider-specific normalization that must populate {@link #backendConfigProperties}. + * Called after binding and {@link #checkRequiredProperties()}. + */ + protected abstract void doInitNormalizeAndCheckProps(); + + // The config directory prefix is taken from the injected `_HADOOP_CONFIG_DIR_` property + // instead of fe-core's Config.hadoop_config_dir, keeping this module fe-core independent. + protected Map loadConfigFromFile(String resourceConfig) { + if (Strings.isNullOrEmpty(resourceConfig)) { + return new HashMap<>(); + } + String configDir = origProps == null ? null : origProps.get("_HADOOP_CONFIG_DIR_"); + return HdfsConfigFileLoader.load(resourceConfig, configDir); + } + + public Map getBackendConfigProperties() { + return backendConfigProperties; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsConfigFileLoader.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsConfigFileLoader.java new file mode 100644 index 00000000000000..49dedf4173f898 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsConfigFileLoader.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs.properties; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +/** + * Loads Hadoop xml config files (hdfs-site.xml / core-site.xml ...) into a key-value map. + * + *

Faithful port of the Hadoop branch of fe-common's {@code CatalogConfigFileUtils}, + * minus the HiveConf path and the direct dependency on fe-core's {@code Config}. The config + * directory prefix is supplied by the caller (sourced from the {@code _HADOOP_CONFIG_DIR_} + * property that fe-core injects) instead of being read from {@code Config.hadoop_config_dir}. + */ +public final class HdfsConfigFileLoader { + + private HdfsConfigFileLoader() { + } + + public static Map load(String resourcesPath, String configDir) { + Map result = new HashMap<>(); + if (StringUtils.isBlank(resourcesPath)) { + return result; + } + String dir = configDir == null ? "" : configDir; + Configuration conf = new Configuration(); + for (String resource : resourcesPath.split(",")) { + String path = dir + resource.trim(); + File file = new File(path); + if (file.exists() && file.isFile()) { + conf.addResource(new Path(file.toURI())); + } else { + throw new IllegalArgumentException("Config resource file does not exist: " + path); + } + } + for (Map.Entry entry : conf) { + result.put(entry.getKey(), entry.getValue()); + } + return result; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesUtils.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesUtils.java new file mode 100644 index 00000000000000..18972d59eb28f5 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesUtils.java @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs.properties; + +import org.apache.doris.foundation.property.StoragePropertiesException; + +import com.google.common.base.Strings; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class HdfsPropertiesUtils { + private static final Logger LOG = LogManager.getLogger(HdfsPropertiesUtils.class); + private static final String URI_KEY = "uri"; + private static final String STANDARD_HDFS_PREFIX = "hdfs://"; + private static final String EMPTY_HDFS_PREFIX = "hdfs:///"; + private static final String BROKEN_HDFS_PREFIX = "hdfs:/"; + private static final String SCHEME_DELIM = "://"; + private static final String NONSTANDARD_SCHEME_DELIM = ":/"; + + public static String validateAndGetUri(Map props, String host, String defaultFs, + Set supportSchemas) throws UserException { + if (props.isEmpty()) { + throw new UserException("props is empty"); + } + String uriStr = getUri(props); + if (StringUtils.isBlank(uriStr)) { + throw new StoragePropertiesException("props must contain uri"); + } + return validateAndNormalizeUri(uriStr, host, defaultFs, supportSchemas); + } + + public static boolean validateUriIsHdfsUri(Map props, + Set supportSchemas) { + String uriStr = getUri(props); + if (StringUtils.isBlank(uriStr)) { + return false; + } + URI uri; + try { + uri = URI.create(uriStr); + } catch (Exception ex) { + // The glob syntax of s3 contains {, which will cause an error here. + LOG.warn("Failed to validate uri is hdfs uri, {}", ex.getMessage()); + return false; + } + String schema = uri.getScheme(); + if (StringUtils.isBlank(schema)) { + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + } + return isSupportedSchema(schema, supportSchemas); + } + + public static String extractDefaultFsFromPath(String filePath) { + if (StringUtils.isBlank(filePath)) { + return null; + } + URI uri = URI.create(filePath); + return uri.getScheme() + "://" + uri.getAuthority(); + } + + public static String extractDefaultFsFromUri(Map props, Set supportSchemas) { + String uriStr = getUri(props); + if (StringUtils.isBlank(uriStr)) { + return null; + } + URI uri = URI.create(uriStr); + if (!isSupportedSchema(uri.getScheme(), supportSchemas)) { + return null; + } + return uri.getScheme() + "://" + uri.getAuthority(); + } + + public static String convertUrlToFilePath(String uriStr, String host, + String defaultFs, Set supportSchemas) { + return validateAndNormalizeUri(uriStr, host, defaultFs, supportSchemas); + } + + public static String convertUrlToFilePath(String uriStr, String host, Set supportSchemas) { + return validateAndNormalizeUri(uriStr, host, null, supportSchemas); + } + + /* + * Extracts the URI value from the given properties. + * If multiple URIs are specified (separated by commas), this method returns null. + * Note: Some storage systems may support multiple URIs (e.g., for load balancing or multi-host), + * but in the HDFS scenario, fs.defaultFS only supports a single URI. + * Therefore, such a format is considered invalid for HDFS. so, just return null. + */ + private static String getUri(Map props) { + String uriValue = props.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) + .map(Map.Entry::getValue) + .filter(StringUtils::isNotBlank) + .findFirst() + .orElse(null); + if (uriValue == null) { + return null; + } + String[] uris = uriValue.split(","); + if (uris.length > 1) { + return null; + } + return uriValue; + } + + private static boolean isSupportedSchema(String schema, Set supportSchema) { + return schema != null && supportSchema.contains(schema.toLowerCase()); + } + + public static String validateAndNormalizeUri(String location, Set supportedSchemas) { + return validateAndNormalizeUri(location, null, null, supportedSchemas); + } + + public static String validateAndNormalizeUri(String location, String host, String defaultFs, + Set supportedSchemas) { + if (StringUtils.isBlank(location)) { + throw new IllegalArgumentException("Property 'uri' is required."); + } + if (!(location.contains(SCHEME_DELIM) || location.contains(NONSTANDARD_SCHEME_DELIM)) + && StringUtils.isNotBlank(defaultFs)) { + location = defaultFs + location; + } + try { + // Encode the location string, but keep '/' and ':' unescaped to preserve URI structure + String newLocation = URLEncoder.encode(location, StandardCharsets.UTF_8.name()) + .replace("%2F", "/") + .replace("%3A", ":"); + + URI uri = new URI(newLocation).normalize(); + + boolean isSupportedSchema = isSupportedSchema(uri.getScheme(), supportedSchemas); + if (!isSupportedSchema) { + throw new IllegalArgumentException("Unsupported schema: " + uri.getScheme()); + } + // compatible with 'hdfs:///' or 'hdfs:/' + if (StringUtils.isEmpty(uri.getHost())) { + newLocation = URLDecoder.decode(newLocation, StandardCharsets.UTF_8.name()); + if (newLocation.startsWith(BROKEN_HDFS_PREFIX) && !newLocation.startsWith(STANDARD_HDFS_PREFIX)) { + newLocation = newLocation.replace(BROKEN_HDFS_PREFIX, STANDARD_HDFS_PREFIX); + } + if (StringUtils.isNotEmpty(host)) { + // Replace 'hdfs://key/' to 'hdfs://name_service/key/' + // Or hdfs:///abc to hdfs://name_service/abc + if (newLocation.startsWith(EMPTY_HDFS_PREFIX)) { + return newLocation.replace(STANDARD_HDFS_PREFIX, STANDARD_HDFS_PREFIX + host); + } else { + return newLocation.replace(STANDARD_HDFS_PREFIX, STANDARD_HDFS_PREFIX + host + "/"); + } + } else { + // 'hdfs://null/' equals the 'hdfs:///' + if (newLocation.startsWith(EMPTY_HDFS_PREFIX)) { + // Do not support hdfs:///location + throw new RuntimeException("Invalid location with empty host: " + newLocation); + } else { + // Replace 'hdfs://key/' to '/key/', try access local NameNode on BE. + return newLocation.replace(STANDARD_HDFS_PREFIX, "/"); + } + } + } + // Normal case: decode and return the fully-qualified URI + return URLDecoder.decode(newLocation, StandardCharsets.UTF_8.name()); + + } catch (URISyntaxException | UnsupportedEncodingException e) { + throw new StoragePropertiesException("Failed to parse URI: " + location, e); + } + } + + /** + * Validate the required HDFS HA configuration properties. + * + *

This method checks the following: + *

    + *
  • {@code dfs.nameservices} must be defined if HA is enabled.
    • + *
  • {@code dfs.ha.namenodes.} must be defined and contain at least 2 namenodes.
    • + *
  • For each namenode, {@code dfs.namenode.rpc-address..} must be defined.
    • + *
  • {@code dfs.client.failover.proxy.provider.} must be defined.
    • + *
+ * + * @param hdfsProperties configuration map (similar to core-site.xml/hdfs-site.xml properties) + */ + public static void checkHaConfig(Map hdfsProperties) { + if (hdfsProperties == null) { + return; + } + // 1. Check dfs.nameservices + String dfsNameservices = hdfsProperties.getOrDefault(HdfsClientConfigKeys.DFS_NAMESERVICES, ""); + if (Strings.isNullOrEmpty(dfsNameservices)) { + // No nameservice configured => HA is not enabled, nothing to validate + return; + } + for (String dfsservice : splitAndTrim(dfsNameservices)) { + if (dfsservice.isEmpty()) { + continue; + } + // 2. Check dfs.ha.namenodes. + String haNnKey = HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + dfsservice; + String namenodes = hdfsProperties.getOrDefault(haNnKey, ""); + if (Strings.isNullOrEmpty(namenodes)) { + throw new IllegalArgumentException("Missing property: " + haNnKey); + } + List names = splitAndTrim(namenodes); + if (names.size() < 2) { + throw new IllegalArgumentException("HA requires at least 2 namenodes for service: " + dfsservice); + } + // 3. Check dfs.namenode.rpc-address.. + for (String name : names) { + String rpcKey = HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + dfsservice + "." + name; + String address = hdfsProperties.getOrDefault(rpcKey, ""); + if (Strings.isNullOrEmpty(address)) { + throw new IllegalArgumentException("Missing property: " + rpcKey + " (expected format: host:port)"); + } + } + // 4. Check dfs.client.failover.proxy.provider. + String failoverKey = HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + dfsservice; + String failoverProvider = hdfsProperties.getOrDefault(failoverKey, ""); + if (Strings.isNullOrEmpty(failoverProvider)) { + throw new IllegalArgumentException("Missing property: " + failoverKey); + } + } + } + + /** + * Utility method to split a comma-separated string, trim whitespace, + * and remove empty tokens. + * + * @param s the input string + * @return list of trimmed non-empty values + */ + private static List splitAndTrim(String s) { + if (Strings.isNullOrEmpty(s)) { + return Collections.emptyList(); + } + return Arrays.stream(s.split(",")) + .map(String::trim) + .filter(tok -> !tok.isEmpty()) + .collect(Collectors.toList()); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/UserException.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/UserException.java new file mode 100644 index 00000000000000..0a3cb9f75d61c6 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/main/java/org/apache/doris/filesystem/hdfs/properties/UserException.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs.properties; + +/** + * Minimal local copy of fe-common's {@code UserException}, kept so the migrated HDFS + * property classes preserve their original {@code throws UserException} signatures without + * depending on fe-common (the original drags in InternalErrorCode and the large ErrorCode + * enum). Only the constructors used by the migrated code are provided. + */ +public class UserException extends Exception { + + public UserException(String msg) { + super(msg); + } + + public UserException(String msg, Throwable cause) { + super(msg, cause); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemEnvTest.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemEnvTest.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemEnvTest.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemEnvTest.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileIteratorTest.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileIteratorTest.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileIteratorTest.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileIteratorTest.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsSeekableInputStreamTest.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/HdfsSeekableInputStreamTest.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsSeekableInputStreamTest.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/HdfsSeekableInputStreamTest.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/KerberosHadoopAuthenticatorEnvTest.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/KerberosHadoopAuthenticatorEnvTest.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/KerberosHadoopAuthenticatorEnvTest.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/KerberosHadoopAuthenticatorEnvTest.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/SimpleHadoopAuthenticatorTest.java b/fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/SimpleHadoopAuthenticatorTest.java similarity index 100% rename from fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/SimpleHadoopAuthenticatorTest.java rename to fe/fe-filesystem/fe-filesystem-hdfs-base/src/test/java/org/apache/doris/filesystem/hdfs/SimpleHadoopAuthenticatorTest.java diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/pom.xml b/fe/fe-filesystem/fe-filesystem-hdfs/pom.xml index a7ff85ea10958d..04f318c9e9bcd7 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/pom.xml +++ b/fe/fe-filesystem/fe-filesystem-hdfs/pom.xml @@ -31,41 +31,27 @@ under the License. fe-filesystem-hdfs jar - Doris FE Filesystem - HDFS / ViewFS / OSS-HDFS - HDFS, ViewFS, JFS, OFS, and OSSHdfs filesystem implementation for Doris FE. + Doris FE Filesystem - HDFS / ViewFS + HDFS and ViewFS filesystem implementation for Doris FE. org.apache.doris fe-filesystem-spi ${revision} + + provided - - - org.apache.hadoop - hadoop-client - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - org.apache.hadoop - hadoop-auth + org.apache.doris + fe-filesystem-hdfs-base + ${revision} - - org.apache.logging.log4j - log4j-api + org.projectlombok + lombok + provided - org.junit.jupiter junit-jupiter diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/assembly/plugin-zip.xml b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/assembly/plugin-zip.xml index e65d143ff9b7d2..c238f9ebb069b6 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/assembly/plugin-zip.xml +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/assembly/plugin-zip.xml @@ -55,11 +55,6 @@ under the License. /lib false runtime - - org.apache.doris:fe-filesystem-api - org.apache.doris:fe-filesystem-spi - org.apache.doris:fe-extension-spi - diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProvider.java index 62e50015f7a55e..f8b7247a518af5 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProvider.java @@ -18,27 +18,36 @@ package org.apache.doris.filesystem.hdfs; import org.apache.doris.filesystem.FileSystem; +import org.apache.doris.filesystem.hdfs.properties.HdfsProperties; import org.apache.doris.filesystem.properties.FileSystemProperties; import org.apache.doris.filesystem.spi.FileSystemProvider; import java.io.IOException; +import java.util.Locale; import java.util.Map; import java.util.Set; /** - * SPI provider for HDFS-family filesystems: hdfs, viewfs, ofs, jfs, oss. + * SPI provider for plain HDFS-family filesystems: hdfs, viewfs only. * Registered via META-INF/services for Java ServiceLoader discovery. + * + *

Aliyun OSS-HDFS ({@code oss://}, JindoFS) is served by the sibling {@code fe-filesystem-oss-hdfs} + * plugin, and JuiceFS ({@code jfs://}) by the sibling {@code fe-filesystem-jfs} plugin; + * {@code ofs://} (Tencent CHDFS) is broker-routed by fe-core and is not an SPI filesystem here. + * Routing is kept strictly disjoint: a concrete uri scheme is authoritative and only + * {@code hdfs}/{@code viewfs} are claimed here; the {@code _STORAGE_TYPE_} "HDFS" marker is only a + * fallback when there is no uri scheme.

*/ public class HdfsFileSystemProvider implements FileSystemProvider { - public static final Set SUPPORTED_SCHEMES = Set.of("hdfs", "viewfs", "ofs", "jfs", "oss"); + public static final Set SUPPORTED_SCHEMES = Set.of("hdfs", "viewfs"); @Override public boolean supports(Map properties) { - // Authoritative match: StoragePropertiesConverter always sets this key for HDFS storage, - // including Hive catalog properties that may not carry explicit HDFS connection keys. - if ("HDFS".equals(properties.get("_STORAGE_TYPE_"))) { - return true; + String storageType = properties.get("_STORAGE_TYPE_"); + // OSS-HDFS (oss://, JindoFS) belongs to OssHdfsFileSystemProvider. + if ("OSS_HDFS".equals(storageType)) { + return false; } String uri = properties.get("HDFS_URI"); if (uri == null) { @@ -47,17 +56,29 @@ public boolean supports(Map properties) { if (uri != null) { int schemeEnd = uri.indexOf("://"); if (schemeEnd > 0) { - String scheme = uri.substring(0, schemeEnd).toLowerCase(); - return SUPPORTED_SCHEMES.contains(scheme); + // A concrete scheme is authoritative: only hdfs/viewfs are ours. jfs:// and oss:// + // are served by sibling filesystem plugins and ofs:// is broker-routed by fe-core, + // so all are declined here even though fe-core also marks jfs as "HDFS". + return SUPPORTED_SCHEMES.contains(uri.substring(0, schemeEnd).toLowerCase(Locale.ROOT)); } } + // No uri scheme: fall back to the authoritative HDFS marker or HA/kerberos hints + // (e.g. Hive catalog properties without explicit connection URIs). + if ("HDFS".equals(storageType)) { + return true; + } return properties.containsKey("dfs.nameservices") || properties.containsKey("hadoop.kerberos.principal"); } @Override public FileSystem create(Map properties) throws IOException { - return new DFSFileSystem(properties); + // Resolve raw user properties through the migrated HdfsProperties so that typed auth + // params (hdfs.authentication.*) are translated to Hadoop keys, xml resources are + // loaded, and defaults are injected — instead of passing raw keys straight through. + HdfsProperties hdfsProperties = new HdfsProperties(properties); + hdfsProperties.initNormalizeAndCheckProps(); + return new DFSFileSystem(hdfsProperties.getBackendConfigProperties()); } @Override diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsProperties.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsProperties.java new file mode 100644 index 00000000000000..9d330eca523acf --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/HdfsProperties.java @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs.properties; + +import org.apache.doris.foundation.property.ConnectorProperty; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Plain HDFS storage properties for fe-filesystem. + * + *

Self-contained port of the kernel {@code HdfsProperties} chain, sitting on the shared + * {@link HdfsCompatibleProperties} base. It binds {@code @ConnectorProperty} fields via + * fe-foundation, translates the typed authentication parameters into Hadoop configuration keys, + * loads xml resources, and injects defaults — with zero fe-core / fe-common dependency.

+ */ +public class HdfsProperties extends HdfsCompatibleProperties { + + private static final Set SUPPORT_SCHEMA = ImmutableSet.of("hdfs", "viewfs"); + + @ConnectorProperty(names = {"hdfs.authentication.type", "hadoop.security.authentication"}, + required = false, + description = "The authentication type of HDFS. The default value is 'none'.") + private String hdfsAuthenticationType = "simple"; + + @ConnectorProperty(names = {"hdfs.authentication.kerberos.principal", "hadoop.kerberos.principal"}, + required = false, + description = "The principal of the kerberos authentication.") + private String hdfsKerberosPrincipal = ""; + + @ConnectorProperty(names = {"hdfs.authentication.kerberos.keytab", "hadoop.kerberos.keytab"}, + required = false, + description = "The keytab of the kerberos authentication.") + private String hdfsKerberosKeytab = ""; + + @ConnectorProperty(names = {"hadoop.username"}, + required = false, + description = "The username of Hadoop. Doris will user this user to access HDFS") + private String hadoopUsername = ""; + + @ConnectorProperty(names = {"hdfs.impersonation.enabled"}, + required = false, + supported = false, + description = "Whether to enable the impersonation of HDFS.") + private boolean hdfsImpersonationEnabled = false; + + @ConnectorProperty(names = {"ipc.client.fallback-to-simple-auth-allowed"}, + required = false, + description = "Whether to allow fallback to simple authentication.") + private String allowFallbackToSimpleAuth = ""; + + @ConnectorProperty(names = {"fs.defaultFS"}, required = false, description = "") + private String fsDefaultFS = ""; + + @ConnectorProperty(names = {"hadoop.config.resources"}, + required = false, + description = "The xml files of Hadoop configuration.") + private String hadoopConfigResources = ""; + + private Map userOverriddenHdfsConfig; + + public HdfsProperties(Map origProps) { + super(origProps); + } + + @Override + protected void checkRequiredProperties() { + super.checkRequiredProperties(); + if ("kerberos".equalsIgnoreCase(hdfsAuthenticationType) && (Strings.isNullOrEmpty(hdfsKerberosPrincipal) + || Strings.isNullOrEmpty(hdfsKerberosKeytab))) { + throw new IllegalArgumentException("HDFS authentication type is kerberos, " + + "but principal or keytab is not set."); + } + } + + @Override + protected void doInitNormalizeAndCheckProps() { + if (StringUtils.isBlank(fsDefaultFS)) { + this.fsDefaultFS = HdfsPropertiesUtils.extractDefaultFsFromUri(origProps, SUPPORT_SCHEMA); + } + extractUserOverriddenHdfsConfig(origProps); + initBackendConfigProperties(); + HdfsPropertiesUtils.checkHaConfig(backendConfigProperties); + } + + private void extractUserOverriddenHdfsConfig(Map origProps) { + if (MapUtils.isEmpty(origProps)) { + return; + } + userOverriddenHdfsConfig = new HashMap<>(); + origProps.forEach((key, value) -> { + if (key.startsWith("hadoop.") || key.startsWith("dfs.") || key.startsWith("fs.") + || key.startsWith("juicefs.")) { + userOverriddenHdfsConfig.put(key, value); + } + }); + } + + private void initBackendConfigProperties() { + Map props = loadConfigFromFile(hadoopConfigResources); + if (MapUtils.isNotEmpty(userOverriddenHdfsConfig)) { + props.putAll(userOverriddenHdfsConfig); + } + if (StringUtils.isNotBlank(fsDefaultFS)) { + props.put(HDFS_DEFAULT_FS_NAME, fsDefaultFS); + } + if (StringUtils.isNotBlank(allowFallbackToSimpleAuth)) { + props.put("ipc.client.fallback-to-simple-auth-allowed", allowFallbackToSimpleAuth); + } else { + props.put("ipc.client.fallback-to-simple-auth-allowed", "true"); + } + props.put("hdfs.security.authentication", hdfsAuthenticationType); + if ("kerberos".equalsIgnoreCase(hdfsAuthenticationType)) { + props.put("hadoop.security.authentication", "kerberos"); + props.put("hadoop.kerberos.principal", hdfsKerberosPrincipal); + props.put("hadoop.kerberos.keytab", hdfsKerberosKeytab); + } + if (StringUtils.isNotBlank(hadoopUsername)) { + props.put("hadoop.username", hadoopUsername); + } + if (StringUtils.isBlank(fsDefaultFS)) { + this.fsDefaultFS = props.getOrDefault(HDFS_DEFAULT_FS_NAME, ""); + } + this.backendConfigProperties = props; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilderTest.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilderTest.java index bd3384d0fe2db7..83560ef38fc75a 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilderTest.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsConfigBuilderTest.java @@ -33,9 +33,9 @@ void buildSetsDisableCache() { } @Test - void build_disablesCacheForAllSupportedSchemes() { + void build_disablesCacheForAllServedSchemes() { Configuration conf = HdfsConfigBuilder.build(Map.of()); - for (String scheme : HdfsFileSystemProvider.SUPPORTED_SCHEMES) { + for (String scheme : HdfsConfigBuilder.CACHE_DISABLE_SCHEMES) { Assertions.assertTrue( conf.getBoolean("fs." + scheme + ".impl.disable.cache", false), "fs." + scheme + ".impl.disable.cache should be true"); @@ -45,6 +45,15 @@ void build_disablesCacheForAllSupportedSchemes() { } } + @Test + void build_disablesCacheForOssHdfsScheme() { + // oss is served by OSS-HDFS (JindoFS) via DFSFileSystem, so its FS cache must be disabled + // even though oss:// routing lives in OssHdfsFileSystemProvider, not SUPPORTED_SCHEMES. + Configuration conf = HdfsConfigBuilder.build(Map.of()); + Assertions.assertTrue(conf.getBoolean("fs.oss.impl.disable.cache", false)); + Assertions.assertTrue(conf.getBoolean("fs.AbstractFileSystem.oss.impl.disable.cache", false)); + } + @Test void buildSetsProvidedProperties() { Configuration conf = HdfsConfigBuilder.build(Map.of( diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProviderTest.java new file mode 100644 index 00000000000000..42f0f2ff15c105 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/HdfsFileSystemProviderTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * Plain HDFS must own only hdfs/viewfs. jfs:// (fe-core marker "HDFS") belongs to the jfs plugin, + * oss:// / OSS_HDFS marker to the oss-hdfs plugin. Routing is first-match-wins over an unordered + * provider list, so these predicates must stay disjoint. + */ +class HdfsFileSystemProviderTest { + + private final HdfsFileSystemProvider provider = new HdfsFileSystemProvider(); + + private Map props(String... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return m; + } + + @Test + void claimsHdfsAndViewfsSchemes() { + Assertions.assertTrue(provider.supports(props("fs.defaultFS", "hdfs://ns"))); + Assertions.assertTrue(provider.supports(props("fs.defaultFS", "viewfs://cluster"))); + } + + @Test + void claimsHdfsMarkerWithoutUri() { + Assertions.assertTrue(provider.supports(props("_STORAGE_TYPE_", "HDFS"))); + } + + @Test + void rejectsJfsScheme() { + // jfs carries fe-core marker "HDFS" but belongs to the jfs plugin. + Assertions.assertFalse(provider.supports(props("_STORAGE_TYPE_", "HDFS", + "fs.defaultFS", "jfs://cluster"))); + } + + @Test + void rejectsOssSchemeAndOssHdfsMarker() { + Assertions.assertFalse(provider.supports(props("fs.defaultFS", "oss://bucket/p"))); + Assertions.assertFalse(provider.supports(props("_STORAGE_TYPE_", "OSS_HDFS"))); + } + + @Test + void rejectsOfsScheme() { + // ofs (Tencent CHDFS) is broker-routed by fe-core, never claimed by this SPI provider. + Assertions.assertFalse(provider.supports(props("fs.defaultFS", "ofs://cluster/p"))); + } + + @Test + void schemeMatchIsCaseInsensitive() { + Assertions.assertTrue(provider.supports(props("fs.defaultFS", "HDFS://ns"))); + Assertions.assertTrue(provider.supports(props("fs.defaultFS", "ViewFS://cluster"))); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesTest.java new file mode 100644 index 00000000000000..65f39c9f63039a --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/HdfsPropertiesTest.java @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs.properties; + +import org.apache.doris.filesystem.hdfs.HdfsConfigBuilder; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +class HdfsPropertiesTest { + + private Map resolve(Map raw) { + HdfsProperties props = new HdfsProperties(raw); + props.initNormalizeAndCheckProps(); + return props.getBackendConfigProperties(); + } + + @Test + void kerberosAuthParamsAreTranslatedToHadoopKeys() { + Map raw = new HashMap<>(); + raw.put("fs.defaultFS", "hdfs://ns"); + raw.put("hdfs.authentication.type", "kerberos"); + raw.put("hdfs.authentication.kerberos.principal", "doris/host@REALM"); + raw.put("hdfs.authentication.kerberos.keytab", "/etc/security/doris.keytab"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("kerberos", resolved.get("hadoop.security.authentication")); + Assertions.assertEquals("doris/host@REALM", resolved.get("hadoop.kerberos.principal")); + Assertions.assertEquals("/etc/security/doris.keytab", resolved.get("hadoop.kerberos.keytab")); + Assertions.assertTrue(HdfsConfigBuilder.isKerberosEnabled(resolved)); + } + + @Test + void simpleAuthInjectsFallbackDefault() { + Map raw = new HashMap<>(); + raw.put("fs.defaultFS", "hdfs://ns"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("true", resolved.get("ipc.client.fallback-to-simple-auth-allowed")); + Assertions.assertFalse(HdfsConfigBuilder.isKerberosEnabled(resolved)); + } + + @Test + void userOverriddenHadoopKeysArePreserved() { + Map raw = new HashMap<>(); + raw.put("fs.defaultFS", "hdfs://ns"); + raw.put("dfs.client.use.datanode.hostname", "true"); + raw.put("hadoop.rpc.protection", "privacy"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("true", resolved.get("dfs.client.use.datanode.hostname")); + Assertions.assertEquals("privacy", resolved.get("hadoop.rpc.protection")); + } + + @Test + void xmlResourcesAreLoadedFromInjectedConfigDir(@TempDir Path tmp) throws Exception { + Path site = tmp.resolve("my-hdfs-site.xml"); + Files.write(site, ("" + + "dfs.custom.keyv1" + + "").getBytes()); + + Map raw = new HashMap<>(); + raw.put("fs.defaultFS", "hdfs://ns"); + raw.put("hadoop.config.resources", "my-hdfs-site.xml"); + raw.put("_HADOOP_CONFIG_DIR_", tmp.toString() + "/"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("v1", resolved.get("dfs.custom.key")); + } + + @Test + void xmlResourcesAreLoadedFromAbsolutePathWhenNoConfigDirInjected(@TempDir Path tmp) throws Exception { + Path site = tmp.resolve("abs-hdfs-site.xml"); + Files.write(site, ("" + + "dfs.custom.abs.keyv2" + + "").getBytes()); + + Map raw = new HashMap<>(); + raw.put("fs.defaultFS", "hdfs://ns"); + // No _HADOOP_CONFIG_DIR_ injected: the resource is given as a full absolute path + // and must load as-is (configDir falls back to ""). + raw.put("hadoop.config.resources", site.toString()); + + Map resolved = resolve(raw); + + Assertions.assertEquals("v2", resolved.get("dfs.custom.abs.key")); + } + + @Test + void translationIsIdempotentOnAlreadyResolvedMap() { + Map raw = new HashMap<>(); + raw.put("fs.defaultFS", "hdfs://ns"); + raw.put("hdfs.authentication.type", "kerberos"); + raw.put("hdfs.authentication.kerberos.principal", "doris/host@REALM"); + raw.put("hdfs.authentication.kerberos.keytab", "/etc/security/doris.keytab"); + + Map once = resolve(raw); + Map twice = resolve(once); + + Assertions.assertEquals("kerberos", twice.get("hadoop.security.authentication")); + Assertions.assertEquals("doris/host@REALM", twice.get("hadoop.kerberos.principal")); + Assertions.assertEquals("/etc/security/doris.keytab", twice.get("hadoop.kerberos.keytab")); + Assertions.assertTrue(HdfsConfigBuilder.isKerberosEnabled(twice)); + } +} diff --git a/fe/be-java-extensions/avro-scanner/pom.xml b/fe/fe-filesystem/fe-filesystem-jfs/pom.xml similarity index 53% rename from fe/be-java-extensions/avro-scanner/pom.xml rename to fe/fe-filesystem/fe-filesystem-jfs/pom.xml index 4f2d706ba36e8c..6fc0f940f7a002 100644 --- a/fe/be-java-extensions/avro-scanner/pom.xml +++ b/fe/fe-filesystem/fe-filesystem-jfs/pom.xml @@ -20,86 +20,59 @@ under the License. + 4.0.0 + - be-java-extensions org.apache.doris + fe-filesystem ${revision} + ../pom.xml - 4.0.0 - - avro-scanner - - 8 - 8 - UTF-8 - + fe-filesystem-jfs + jar + Doris FE Filesystem - JFS (JuiceFS) + JuiceFS (jfs://) HDFS-compatible filesystem provider for Doris FE. org.apache.doris - java-common - ${project.version} + fe-filesystem-spi + ${revision} provided - org.apache.thrift - libthrift - - - org.apache.avro - avro-mapred - ${avro.version} - - - org.apache.avro - avro - - - org.apache.avro - avro-tools - - - - - org.apache.hadoop - hadoop-client - provided + org.apache.doris + fe-filesystem-hdfs-base + ${revision} - org.apache.hadoop - hadoop-common + org.projectlombok + lombok provided - org.apache.hadoop - hadoop-hdfs - provided + org.junit.jupiter + junit-jupiter + test - org.apache.doris - hive-catalog-shade - provided + org.mockito + mockito-core + test - - avro-scanner - ${project.basedir}/target/ + doris-fe-filesystem-jfs - org.apache.maven.plugins maven-assembly-plugin + false - src/main/resources/package.xml + src/main/assembly/plugin-zip.xml - - - - - diff --git a/fe/fe-filesystem/fe-filesystem-jfs/src/main/assembly/plugin-zip.xml b/fe/fe-filesystem/fe-filesystem-jfs/src/main/assembly/plugin-zip.xml new file mode 100644 index 00000000000000..0b9a3bf90626a6 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-jfs/src/main/assembly/plugin-zip.xml @@ -0,0 +1,59 @@ + + + + + plugin + + zip + + false + + + + + ${project.build.directory}/${project.build.finalName}.jar + / + + + + + + + /lib + false + runtime + + + diff --git a/fe/fe-filesystem/fe-filesystem-jfs/src/main/java/org/apache/doris/filesystem/jfs/JfsFileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-jfs/src/main/java/org/apache/doris/filesystem/jfs/JfsFileSystemProvider.java new file mode 100644 index 00000000000000..3213d4e5c26315 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-jfs/src/main/java/org/apache/doris/filesystem/jfs/JfsFileSystemProvider.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.jfs; + +import org.apache.doris.filesystem.FileSystem; +import org.apache.doris.filesystem.hdfs.DFSFileSystem; +import org.apache.doris.filesystem.jfs.properties.JfsProperties; +import org.apache.doris.filesystem.properties.FileSystemProperties; +import org.apache.doris.filesystem.spi.FileSystemProvider; + +import java.io.IOException; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * SPI provider for JuiceFS ({@code jfs://}), served by the HDFS-compatible {@code DFSFileSystem}. + * Routing is by URI scheme only; fe-core marks jfs as the generic "HDFS" storage type, so scheme + * is what keeps this disjoint from {@code HdfsFileSystemProvider}. + */ +public class JfsFileSystemProvider implements FileSystemProvider { + + private static final Set SUPPORTED_SCHEMES = Set.of("jfs"); + + @Override + public boolean supports(Map properties) { + String uri = properties.get("HDFS_URI"); + if (uri == null) { + uri = properties.get("fs.defaultFS"); + } + if (uri == null) { + return false; + } + int schemeEnd = uri.indexOf("://"); + if (schemeEnd <= 0) { + return false; + } + return SUPPORTED_SCHEMES.contains(uri.substring(0, schemeEnd).toLowerCase(Locale.ROOT)); + } + + @Override + public FileSystem create(Map properties) throws IOException { + JfsProperties jfsProperties = new JfsProperties(properties); + jfsProperties.initNormalizeAndCheckProps(); + return new DFSFileSystem(jfsProperties.getBackendConfigProperties()); + } + + @Override + public String name() { + return "JFS"; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-jfs/src/main/java/org/apache/doris/filesystem/jfs/properties/JfsProperties.java b/fe/fe-filesystem/fe-filesystem-jfs/src/main/java/org/apache/doris/filesystem/jfs/properties/JfsProperties.java new file mode 100644 index 00000000000000..319c237da30aad --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-jfs/src/main/java/org/apache/doris/filesystem/jfs/properties/JfsProperties.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.jfs.properties; + +import org.apache.doris.filesystem.hdfs.properties.HdfsCompatibleProperties; +import org.apache.doris.filesystem.hdfs.properties.HdfsPropertiesUtils; +import org.apache.doris.foundation.property.ConnectorProperty; + +import com.google.common.collect.ImmutableSet; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * JuiceFS ({@code jfs://}) storage properties. HDFS-compatible: JuiceFS ships a Hadoop FileSystem + * impl resolved by Hadoop config, so this only needs to derive {@code fs.defaultFS} from a + * {@code jfs://} uri and forward the {@code juicefs.*} / Hadoop keys to {@code DFSFileSystem}. + */ +public class JfsProperties extends HdfsCompatibleProperties { + + private static final Set SUPPORT_SCHEMA = ImmutableSet.of("jfs"); + + @ConnectorProperty(names = {"fs.defaultFS"}, required = false, description = "") + private String fsDefaultFS = ""; + + @ConnectorProperty(names = {"hadoop.config.resources"}, required = false, + description = "The xml files of Hadoop configuration.") + private String hadoopConfigResources = ""; + + @ConnectorProperty(names = {"hadoop.username"}, required = false, + description = "The username used to access JuiceFS.") + private String hadoopUsername = ""; + + private Map userOverriddenConfig; + + public JfsProperties(Map origProps) { + super(origProps); + } + + @Override + protected void doInitNormalizeAndCheckProps() { + if (StringUtils.isBlank(fsDefaultFS)) { + this.fsDefaultFS = HdfsPropertiesUtils.extractDefaultFsFromUri(origProps, SUPPORT_SCHEMA); + } + extractUserOverriddenConfig(origProps); + initBackendConfigProperties(); + } + + private void extractUserOverriddenConfig(Map origProps) { + if (MapUtils.isEmpty(origProps)) { + return; + } + userOverriddenConfig = new HashMap<>(); + origProps.forEach((key, value) -> { + if (key.startsWith("hadoop.") || key.startsWith("dfs.") || key.startsWith("fs.") + || key.startsWith("juicefs.")) { + userOverriddenConfig.put(key, value); + } + }); + } + + private void initBackendConfigProperties() { + Map props = loadConfigFromFile(hadoopConfigResources); + if (MapUtils.isNotEmpty(userOverriddenConfig)) { + props.putAll(userOverriddenConfig); + } + if (StringUtils.isNotBlank(fsDefaultFS)) { + props.put(HDFS_DEFAULT_FS_NAME, fsDefaultFS); + } + if (StringUtils.isNotBlank(hadoopUsername)) { + props.put("hadoop.username", hadoopUsername); + } + this.backendConfigProperties = props; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-jfs/src/main/resources/META-INF/services/org.apache.doris.filesystem.spi.FileSystemProvider b/fe/fe-filesystem/fe-filesystem-jfs/src/main/resources/META-INF/services/org.apache.doris.filesystem.spi.FileSystemProvider new file mode 100644 index 00000000000000..1ebbbf528dd7bf --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-jfs/src/main/resources/META-INF/services/org.apache.doris.filesystem.spi.FileSystemProvider @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +org.apache.doris.filesystem.jfs.JfsFileSystemProvider diff --git a/fe/fe-filesystem/fe-filesystem-jfs/src/test/java/org/apache/doris/filesystem/jfs/JfsFileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-jfs/src/test/java/org/apache/doris/filesystem/jfs/JfsFileSystemProviderTest.java new file mode 100644 index 00000000000000..52b42544343c3f --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-jfs/src/test/java/org/apache/doris/filesystem/jfs/JfsFileSystemProviderTest.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.jfs; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class JfsFileSystemProviderTest { + + private final JfsFileSystemProvider provider = new JfsFileSystemProvider(); + + private Map props(String... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return m; + } + + @Test + void claimsJfsScheme() { + Assertions.assertTrue(provider.supports(props("fs.defaultFS", "jfs://myvol"))); + } + + @Test + void rejectsOtherSchemesAndMarkers() { + Assertions.assertFalse(provider.supports(props("fs.defaultFS", "hdfs://ns"))); + Assertions.assertFalse(provider.supports(props("fs.defaultFS", "viewfs://c"))); + Assertions.assertFalse(provider.supports(props("fs.defaultFS", "oss://bucket/p"))); + Assertions.assertFalse(provider.supports(props("_STORAGE_TYPE_", "HDFS"))); + } + + @Test + void claimsJfsViaHdfsUriKey() { + // fe-core injects the connection URI under HDFS_URI; supports() must honor that key too. + Assertions.assertTrue(provider.supports(props("HDFS_URI", "jfs://myvol"))); + } + + @Test + void rejectsUriWithoutScheme() { + Assertions.assertFalse(provider.supports(props("fs.defaultFS", "myvol"))); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-jfs/src/test/java/org/apache/doris/filesystem/jfs/properties/JfsPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-jfs/src/test/java/org/apache/doris/filesystem/jfs/properties/JfsPropertiesTest.java new file mode 100644 index 00000000000000..318bbe9b22be6c --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-jfs/src/test/java/org/apache/doris/filesystem/jfs/properties/JfsPropertiesTest.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.jfs.properties; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class JfsPropertiesTest { + + private Map resolve(Map raw) { + JfsProperties props = new JfsProperties(raw); + props.initNormalizeAndCheckProps(); + return props.getBackendConfigProperties(); + } + + @Test + void jfsUriDerivesDefaultFs() { + Map raw = new HashMap<>(); + raw.put("uri", "jfs://myvol/path/to/file"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("jfs://myvol", resolved.get("fs.defaultFS")); + } + + @Test + void juicefsKeysArePassedThrough() { + Map raw = new HashMap<>(); + raw.put("fs.defaultFS", "jfs://myvol"); + raw.put("juicefs.meta", "redis://localhost:6379/0"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("redis://localhost:6379/0", resolved.get("juicefs.meta")); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsFileSystem.java b/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsFileSystem.java index 69fa992e936a6c..6d4190eeb03071 100644 --- a/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsFileSystem.java @@ -25,6 +25,6 @@ public class ObsFileSystem extends S3CompatibleFileSystem { public ObsFileSystem(ObsObjStorage objStorage) { - super(objStorage, objStorage.isUsePathStyle()); + super(objStorage, objStorage.isUsePathStyle(), objStorage.getSupportedSchemes()); } } diff --git a/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsFileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsFileSystemProperties.java index a0c171381da1b9..795555f148b9f7 100644 --- a/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsFileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsFileSystemProperties.java @@ -36,6 +36,7 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -349,6 +350,11 @@ public String getUsePathStyle() { return usePathStyle; } + @Override + public Set getSupportedSchemes() { + return Set.of("obs", "s3", "s3a"); + } + public String getForceParsingByStandardUrl() { return forceParsingByStandardUrl; } diff --git a/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsObjStorage.java b/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsObjStorage.java index caf3ea48534aac..ca40180e07e0fa 100644 --- a/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-obs/src/main/java/org/apache/doris/filesystem/obs/ObsObjStorage.java @@ -68,6 +68,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -102,6 +103,11 @@ public boolean isUsePathStyle() { return properties.isUsePathStyle(); } + /** Returns the URI schemes this provider accepts (e.g. {@code {obs, s3, s3a}}). */ + public Set getSupportedSchemes() { + return properties.getSupportedSchemes(); + } + @Override public ObsClient getClient() throws IOException { if (closed.get()) { @@ -141,7 +147,7 @@ public RemoteObjects listObjects(String remotePath, String continuationToken) th @Override public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions options) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); ListObjectsRequest request = new ListObjectsRequest(uri.bucket()); request.setPrefix(uri.key()); if (options != null) { @@ -171,7 +177,7 @@ public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions @Override public RemoteObject headObject(String remotePath) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { ObjectMetadata metadata = getClient().getObjectMetadata(uri.bucket(), uri.key()); return new RemoteObject(uri.key(), uri.key(), metadata.getEtag(), contentLength(metadata), @@ -186,7 +192,7 @@ public RemoteObject headObject(String remotePath) throws IOException { @Override public void putObject(String remotePath, RequestBody requestBody) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try (InputStream content = requestBody.content()) { PutObjectRequest request = new PutObjectRequest(uri.bucket(), uri.key(), content); ObjectMetadata metadata = new ObjectMetadata(); @@ -201,7 +207,7 @@ public void putObject(String remotePath, RequestBody requestBody) throws IOExcep @Override public void deleteObject(String remotePath) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { getClient().deleteObject(uri.bucket(), uri.key()); } catch (ObsException e) { @@ -214,8 +220,8 @@ public void deleteObject(String remotePath) throws IOException { @Override public void copyObject(String srcPath, String dstPath) throws IOException { - ObjectStorageUri src = ObjectStorageUri.parse(srcPath, false); - ObjectStorageUri dst = ObjectStorageUri.parse(dstPath, false); + ObjectStorageUri src = ObjectStorageUri.parse(srcPath, isUsePathStyle(), getSupportedSchemes()); + ObjectStorageUri dst = ObjectStorageUri.parse(dstPath, isUsePathStyle(), getSupportedSchemes()); try { getClient().copyObject(new CopyObjectRequest( src.bucket(), src.key(), dst.bucket(), dst.key())); @@ -227,7 +233,7 @@ public void copyObject(String srcPath, String dstPath) throws IOException { @Override public String initiateMultipartUpload(String remotePath) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { InitiateMultipartUploadResult result = getClient().initiateMultipartUpload( new InitiateMultipartUploadRequest(uri.bucket(), uri.key())); @@ -241,7 +247,7 @@ public String initiateMultipartUpload(String remotePath) throws IOException { @Override public UploadPartResult uploadPart(String remotePath, String uploadId, int partNum, RequestBody body) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try (InputStream content = body.content()) { com.obs.services.model.UploadPartRequest request = new com.obs.services.model.UploadPartRequest(uri.bucket(), uri.key()); @@ -261,7 +267,7 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN @Override public void completeMultipartUpload(String remotePath, String uploadId, List parts) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); List partEtags = parts.stream() .map(part -> new PartEtag(part.etag(), part.partNumber())) .collect(Collectors.toList()); @@ -276,7 +282,7 @@ public void completeMultipartUpload(String remotePath, String uploadId, @Override public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { getClient().abortMultipartUpload(new AbortMultipartUploadRequest( uri.bucket(), uri.key(), uploadId)); @@ -288,7 +294,7 @@ public void abortMultipartUpload(String remotePath, String uploadId) throws IOEx @Override public InputStream openInputStreamAt(String remotePath, long fromByte) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { GetObjectRequest request = new GetObjectRequest(uri.bucket(), uri.key()); if (fromByte > 0) { @@ -382,7 +388,7 @@ public String getPresignedUrl(String objectKey) throws IOException { request.setHeaders(new HashMap<>()); TemporarySignatureResponse response = getClient().createTemporarySignature(request); String url = response.getSignedUrl(); - LOG.info("Generated OBS temporary signature URL for key={}", objectKey); + LOG.debug("Generated OBS temporary signature URL for key={}", objectKey); return url; } catch (ObsException e) { throw new IOException("Failed to generate OBS presigned URL: " + e.getMessage(), e); diff --git a/fe/fe-filesystem/fe-filesystem-oss-hdfs/pom.xml b/fe/fe-filesystem/fe-filesystem-oss-hdfs/pom.xml new file mode 100644 index 00000000000000..dbdb24ee472f60 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss-hdfs/pom.xml @@ -0,0 +1,89 @@ + + + + 4.0.0 + + + org.apache.doris + fe-filesystem + ${revision} + ../pom.xml + + + fe-filesystem-oss-hdfs + jar + Doris FE Filesystem - OSS-HDFS (JindoFS) + Aliyun OSS-HDFS (JindoFS) HDFS-compatible filesystem provider for Doris FE. + + + + org.apache.doris + fe-filesystem-spi + ${revision} + provided + + + org.apache.doris + fe-filesystem-hdfs-base + ${revision} + + + org.projectlombok + lombok + provided + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-core + test + + + + + doris-fe-filesystem-oss-hdfs + + + maven-assembly-plugin + + false + + src/main/assembly/plugin-zip.xml + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/assembly/plugin-zip.xml b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/assembly/plugin-zip.xml new file mode 100644 index 00000000000000..0b9a3bf90626a6 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/assembly/plugin-zip.xml @@ -0,0 +1,59 @@ + + + + + plugin + + zip + + false + + + + + ${project.build.directory}/${project.build.finalName}.jar + / + + + + + + + /lib + false + runtime + + + diff --git a/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/OssHdfsFileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/OssHdfsFileSystemProvider.java new file mode 100644 index 00000000000000..c11e59fb50f01b --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/OssHdfsFileSystemProvider.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs; + +import org.apache.doris.filesystem.FileSystem; +import org.apache.doris.filesystem.hdfs.properties.OssHdfsProperties; +import org.apache.doris.filesystem.properties.FileSystemProperties; +import org.apache.doris.filesystem.spi.FileSystemProvider; +import org.apache.doris.foundation.property.ConnectorPropertiesUtils; + +import java.io.IOException; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * SPI provider for Aliyun OSS-HDFS (JindoFS), addressed with {@code oss://} URIs and served by an + * HDFS-compatible {@code DFSFileSystem}. Registered via META-INF/services alongside + * {@link HdfsFileSystemProvider}. + * + *

Routing is kept strictly disjoint from plain HDFS: the authoritative {@code _STORAGE_TYPE_} + * marker set by fe-core's StoragePropertiesConverter wins, and the heuristic fallback keys on + * {@link OssHdfsProperties#guessIsMe} (oss.hdfs flag / oss-dls endpoint) or an {@code oss://} URI. + * The framework selects providers first-match-wins over an unordered list, so the two providers' + * {@code supports()} predicates must not overlap.

+ */ +public class OssHdfsFileSystemProvider implements FileSystemProvider { + + private static final String FS_OSS_HDFS_SUPPORT = "fs.oss-hdfs.support"; + private static final String DEPRECATED_OSS_HDFS_SUPPORT = "oss.hdfs.enabled"; + private static final String FS_OSS_SUPPORT = "fs.oss.support"; + + @Override + public boolean supports(Map properties) { + String storageType = properties.get("_STORAGE_TYPE_"); + if ("OSS_HDFS".equals(storageType)) { + return true; + } + if ("HDFS".equals(storageType)) { + // An explicit plain-HDFS marker belongs to HdfsFileSystemProvider, never here. + return false; + } + if ("OSS".equals(storageType)) { + // A native-OSS (S3-compatible) marker belongs to OssFileSystemProvider, never here. + return false; + } + // Explicit kernel flags, mirroring StorageProperties.createPrimary precedence: + // fs.oss-hdfs.support / oss.hdfs.enabled declare OSS-HDFS and win over everything below, + // while fs.oss.support declares native OSS, so the bare oss:// URI fallback must yield. + if (isFlagTrue(properties, FS_OSS_HDFS_SUPPORT) || isFlagTrue(properties, DEPRECATED_OSS_HDFS_SUPPORT)) { + return true; + } + if (isFlagTrue(properties, FS_OSS_SUPPORT)) { + return false; + } + // No authoritative marker: fall back to the same detection fe-core uses to pick + // OSSHdfsProperties, plus a bare oss:// URI. + if (OssHdfsProperties.guessIsMe(properties)) { + return true; + } + String uri = properties.get("HDFS_URI"); + if (uri == null) { + uri = properties.get("fs.defaultFS"); + } + return uri != null && uri.toLowerCase(Locale.ROOT).startsWith("oss://"); + } + + @Override + public FileSystem create(Map properties) throws IOException { + OssHdfsProperties ossHdfsProperties = new OssHdfsProperties(properties); + ossHdfsProperties.initNormalizeAndCheckProps(); + return new DFSFileSystem(ossHdfsProperties.getBackendConfigProperties()); + } + + @Override + public String name() { + return "OSS_HDFS"; + } + + @Override + public Set sensitivePropertyKeys() { + return ConnectorPropertiesUtils.getSensitiveKeys(OssHdfsProperties.class); + } + + private static boolean isFlagTrue(Map properties, String key) { + return Boolean.parseBoolean(properties.getOrDefault(key, "false")); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/OssHdfsProperties.java b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/OssHdfsProperties.java new file mode 100644 index 00000000000000..66beddf1901760 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/properties/OssHdfsProperties.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs.properties; + +import org.apache.doris.foundation.property.ConnectorProperty; + +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Aliyun OSS-HDFS (JindoFS) storage properties for fe-filesystem. + * + *

OSS-HDFS is an HDFS-compatible service over OSS, addressed with {@code oss://} URIs and + * served by the JindoFS Hadoop FileSystem ({@code com.aliyun.jindodata.oss.JindoOssFileSystem}). + * It therefore sits on the shared {@link HdfsCompatibleProperties} base alongside plain + * {@link HdfsProperties} and produces a backend config map for {@code DFSFileSystem}; the only + * delta is the Jindo {@code fs.oss.*} wiring plus oss-dls endpoint/region handling.

+ * + *

Self-contained port of the kernel {@code OSSHdfsProperties}, with zero fe-core / fe-common + * dependency. The Jindo impl class names are kept as string constants (loaded reflectively by the + * backend), so no compile dependency on the Jindo SDK is introduced.

+ * + *

The normalized {@code fs.oss.*} backend keys are accepted as aliases so the binder is a + * fixpoint for converter-produced maps: fe-core's StoragePropertiesConverter hands the plugin + * {@code getBackendConfigProperties()} output (plus the {@code _STORAGE_TYPE_} marker), not the + * raw user keys, and rebinding that map must yield the same configuration.

+ */ +public class OssHdfsProperties extends HdfsCompatibleProperties { + + private static final String JINDO_OSS_FILE_SYSTEM_IMPL = + "com.aliyun.jindodata.oss.JindoOssFileSystem"; + private static final String JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL = + "com.aliyun.jindodata.oss.JindoOSS"; + + private static final String OSS_HDFS_PREFIX_KEY = "oss.hdfs."; + private static final String OSS_HDFS_ENDPOINT_SUFFIX = ".oss-dls.aliyuncs.com"; + + private static final Set OSS_ENDPOINT_KEY_NAME = ImmutableSet.of("oss.hdfs.endpoint", + "oss.endpoint", "dlf.endpoint", "dlf.catalog.endpoint"); + + private static final Set ENDPOINT_PATTERN = ImmutableSet.of( + Pattern.compile("(?:https?://)?([a-z]{2}-[a-z0-9-]+)\\.oss-dls\\.aliyuncs\\.com"), + Pattern.compile("^(?:https?://)?dlf(?:-vpc)?\\.([a-z0-9-]+)\\.aliyuncs\\.com(?:/.*)?$")); + + private static final Set SUPPORT_SCHEMA = ImmutableSet.of("oss", "hdfs"); + + @ConnectorProperty(names = {"oss.hdfs.endpoint", "oss.endpoint", "dlf.endpoint", "dlf.catalog.endpoint", + "fs.oss.endpoint"}, + description = "The endpoint of OSS.") + private String endpoint = ""; + + @ConnectorProperty(names = {"oss.hdfs.access_key", "oss.access_key", "dlf.access_key", "dlf.catalog.accessKeyId", + "fs.oss.accessKeyId"}, + sensitive = true, + description = "The access key of OSS.") + private String accessKey = ""; + + @ConnectorProperty(names = {"oss.hdfs.secret_key", "oss.secret_key", "dlf.secret_key", "dlf.catalog.secret_key", + "fs.oss.accessKeySecret"}, + sensitive = true, + description = "The secret key of OSS.") + private String secretKey = ""; + + @ConnectorProperty(names = {"oss.hdfs.region", "oss.region", "dlf.region", "fs.oss.region"}, + required = false, + description = "The region of OSS.") + private String region = ""; + + @ConnectorProperty(names = {"oss.hdfs.fs.defaultFS", "fs.defaultFS"}, required = false, description = "") + private String fsDefaultFS = ""; + + @ConnectorProperty(names = {"oss.hdfs.hadoop.config.resources"}, + required = false, + description = "The xml files of Hadoop configuration.") + private String hadoopConfigResources = ""; + + @ConnectorProperty(names = {"oss.hdfs.security_token", "oss.security_token", "fs.oss.securityToken"}, + required = false, + sensitive = true, + description = "The security token of OSS.") + private String securityToken = ""; + + public OssHdfsProperties(Map origProps) { + super(origProps); + } + + /** + * Cheap, deterministic detection of an OSS-HDFS configuration: an explicit {@code oss.hdfs.} + * enable flag, or any endpoint key pointing at an {@code *.oss-dls.aliyuncs.com} host. + */ + public static boolean guessIsMe(Map props) { + boolean enable = props.entrySet().stream() + .anyMatch(e -> e.getKey().equalsIgnoreCase(OSS_HDFS_PREFIX_KEY) + && Boolean.parseBoolean(e.getValue())); + if (enable) { + return true; + } + return OSS_ENDPOINT_KEY_NAME.stream() + .map(props::get) + .anyMatch(ep -> StringUtils.isNotBlank(ep) && ep.endsWith(OSS_HDFS_ENDPOINT_SUFFIX)); + } + + static Optional extractRegion(String endpoint) { + for (Pattern pattern : ENDPOINT_PATTERN) { + Matcher matcher = pattern.matcher(endpoint.toLowerCase()); + if (matcher.matches()) { + return Optional.ofNullable(matcher.group(1)); + } + } + return Optional.empty(); + } + + private void convertDlfToOssEndpointIfNeeded() { + if (this.endpoint.contains("dlf")) { + this.endpoint = this.region + ".oss-dls.aliyuncs.com"; + } + } + + @Override + protected void doInitNormalizeAndCheckProps() { + // Derive region from the endpoint when not explicitly set, e.g. + // "cn-shanghai.oss-dls.aliyuncs.com" -> "cn-shanghai". + if (StringUtils.isBlank(this.region)) { + Optional regionOptional = extractRegion(endpoint); + if (!regionOptional.isPresent()) { + throw new IllegalArgumentException("The region extracted from the endpoint is empty. " + + "Please check the endpoint format: " + endpoint + " or set oss.hdfs.region"); + } + this.region = regionOptional.get(); + } + convertDlfToOssEndpointIfNeeded(); + if (StringUtils.isBlank(fsDefaultFS)) { + this.fsDefaultFS = HdfsPropertiesUtils.extractDefaultFsFromUri(origProps, SUPPORT_SCHEMA); + } + initConfigurationParams(); + } + + private void initConfigurationParams() { + Map config = loadConfigFromFile(hadoopConfigResources); + // Converter-produced backend maps carry XML-loaded tuning keys (e.g. fs.oss.* Jindo + // settings) and fs.defaultFS as plain entries — the config-resources key itself is not + // part of that map — so pass Hadoop-shaped entries through like HdfsProperties and + // JfsProperties do. The derived keys below still win. + origProps.forEach((key, value) -> { + if (key.startsWith("hadoop.") || key.startsWith("dfs.") || key.startsWith("fs.")) { + config.put(key, value); + } + }); + config.put("fs.oss.endpoint", endpoint); + config.put("fs.oss.accessKeyId", accessKey); + config.put("fs.oss.accessKeySecret", secretKey); + config.put("fs.oss.region", region); + config.put("fs.oss.impl", JINDO_OSS_FILE_SYSTEM_IMPL); + config.put("fs.AbstractFileSystem.oss.impl", JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL); + if (StringUtils.isNotBlank(securityToken)) { + config.put("fs.oss.securityToken", securityToken); + } + if (StringUtils.isNotBlank(fsDefaultFS)) { + config.put(HDFS_DEFAULT_FS_NAME, fsDefaultFS); + } + this.backendConfigProperties = config; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/resources/META-INF/services/org.apache.doris.filesystem.spi.FileSystemProvider b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/resources/META-INF/services/org.apache.doris.filesystem.spi.FileSystemProvider new file mode 100644 index 00000000000000..ca3bd75232af49 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/main/resources/META-INF/services/org.apache.doris.filesystem.spi.FileSystemProvider @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +org.apache.doris.filesystem.hdfs.OssHdfsFileSystemProvider diff --git a/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/OssHdfsFileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/OssHdfsFileSystemProviderTest.java new file mode 100644 index 00000000000000..ae066f4b705948 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/OssHdfsFileSystemProviderTest.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class OssHdfsFileSystemProviderTest { + + private final OssHdfsFileSystemProvider provider = new OssHdfsFileSystemProvider(); + + private Map props(String... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return m; + } + + @Test + void claimsOssHdfsMarkerAndOssScheme() { + Assertions.assertTrue(provider.supports(props("_STORAGE_TYPE_", "OSS_HDFS"))); + Assertions.assertTrue(provider.supports(props("fs.defaultFS", "oss://bucket/p"))); + } + + @Test + void claimsOssDlsEndpointWithoutMarker() { + Assertions.assertTrue(provider.supports(props("oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"))); + } + + @Test + void rejectsHdfsAndJfsAndHdfsMarker() { + Assertions.assertFalse(provider.supports(props("fs.defaultFS", "hdfs://ns"))); + Assertions.assertFalse(provider.supports(props("fs.defaultFS", "jfs://cluster"))); + Assertions.assertFalse(provider.supports(props("_STORAGE_TYPE_", "HDFS"))); + } + + @Test + void rejectsNativeOssMarker() { + // A native-OSS (S3-compatible) marker belongs to OssFileSystemProvider, never here. + Assertions.assertFalse(provider.supports(props("_STORAGE_TYPE_", "OSS"))); + } + + @Test + void claimsExplicitOssHdfsFlagsLikeKernel() { + // StorageProperties.createPrimary treats these flags as OSS-HDFS before native OSS, + // even with a plain (non-dls) Aliyun endpoint. + Assertions.assertTrue(provider.supports(props( + "fs.oss-hdfs.support", "true", "oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"))); + Assertions.assertTrue(provider.supports(props( + "oss.hdfs.enabled", "true", "oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"))); + } + + @Test + void yieldsExplicitNativeOssEvenWithOssUri() { + // fs.oss.support=true declares native OSS; the bare oss:// URI fallback must not steal it. + Assertions.assertFalse(provider.supports(props( + "fs.oss.support", "true", "fs.defaultFS", "oss://bucket/p"))); + } + + @Test + void ossHdfsFlagsWinOverNativeOssFlag() { + // Kernel precedence: the OSS-HDFS flags are checked before fs.oss.support. + Assertions.assertTrue(provider.supports(props( + "fs.oss-hdfs.support", "true", "fs.oss.support", "true"))); + } + + @Test + void sensitivePropertyKeysCoverSecretsButNotEndpoint() { + java.util.Set keys = provider.sensitivePropertyKeys(); + Assertions.assertTrue(keys.contains("oss.hdfs.access_key")); + Assertions.assertTrue(keys.contains("oss.hdfs.secret_key")); + Assertions.assertTrue(keys.contains("oss.hdfs.security_token")); + Assertions.assertTrue(keys.contains("oss.security_token")); + Assertions.assertTrue(keys.contains("fs.oss.accessKeySecret")); + Assertions.assertFalse(keys.contains("oss.hdfs.endpoint")); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/OssHdfsPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/OssHdfsPropertiesTest.java new file mode 100644 index 00000000000000..ae38ae87ee573d --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/properties/OssHdfsPropertiesTest.java @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.hdfs.properties; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class OssHdfsPropertiesTest { + + private Map resolve(Map raw) { + OssHdfsProperties props = new OssHdfsProperties(raw); + props.initNormalizeAndCheckProps(); + return props.getBackendConfigProperties(); + } + + private Map baseProps() { + Map raw = new HashMap<>(); + raw.put("oss.hdfs.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + raw.put("oss.hdfs.access_key", "ak"); + raw.put("oss.hdfs.secret_key", "sk"); + return raw; + } + + @Test + void jindoImplAndCredentialsArePopulated() { + Map resolved = resolve(baseProps()); + + Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem", + resolved.get("fs.oss.impl")); + Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOSS", + resolved.get("fs.AbstractFileSystem.oss.impl")); + Assertions.assertEquals("ak", resolved.get("fs.oss.accessKeyId")); + Assertions.assertEquals("sk", resolved.get("fs.oss.accessKeySecret")); + Assertions.assertEquals("cn-beijing.oss-dls.aliyuncs.com", resolved.get("fs.oss.endpoint")); + } + + @Test + void regionIsExtractedFromOssDlsEndpoint() { + Map raw = baseProps(); + raw.put("oss.hdfs.endpoint", "cn-shanghai.oss-dls.aliyuncs.com"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("cn-shanghai", resolved.get("fs.oss.region")); + } + + @Test + void explicitRegionIsHonored() { + Map raw = baseProps(); + raw.put("oss.hdfs.region", "cn-hangzhou"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("cn-hangzhou", resolved.get("fs.oss.region")); + } + + @Test + void unparseableEndpointWithoutRegionThrows() { + Map raw = baseProps(); + raw.put("oss.hdfs.endpoint", "not-a-known-host.example.com"); + + Assertions.assertThrows(IllegalArgumentException.class, () -> resolve(raw)); + } + + /** + * fe-core's StoragePropertiesConverter hands the plugin the normalized backend map + * ({@code fs.oss.*} keys), not the raw user keys; rebinding it must yield the same config. + */ + @Test + void convertedBackendMapRoundTrips() { + Map raw = new HashMap<>(); + raw.put("fs.oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + raw.put("fs.oss.accessKeyId", "ak"); + raw.put("fs.oss.accessKeySecret", "sk"); + raw.put("fs.oss.region", "cn-beijing"); + raw.put("_STORAGE_TYPE_", "OSS_HDFS"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("cn-beijing.oss-dls.aliyuncs.com", resolved.get("fs.oss.endpoint")); + Assertions.assertEquals("ak", resolved.get("fs.oss.accessKeyId")); + Assertions.assertEquals("sk", resolved.get("fs.oss.accessKeySecret")); + Assertions.assertEquals("cn-beijing", resolved.get("fs.oss.region")); + } + + /** + * The converter map also carries entries the kernel loaded from XML config files (Jindo + * tuning keys) plus {@code fs.defaultFS}; rebinding must pass them through instead of + * rebuilding the config from the fixed credential/endpoint fields only. + */ + @Test + void convertedBackendMapPreservesPassThroughEntries() { + Map raw = new HashMap<>(); + raw.put("fs.oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + raw.put("fs.oss.accessKeyId", "ak"); + raw.put("fs.oss.accessKeySecret", "sk"); + raw.put("fs.oss.region", "cn-beijing"); + raw.put("fs.oss.tmp.data.dirs", "/tmp/jindo"); + raw.put("fs.oss.upload.thread.concurrency", "16"); + raw.put("fs.defaultFS", "oss://bucket"); + raw.put("hadoop.username", "hive"); + raw.put("_STORAGE_TYPE_", "OSS_HDFS"); + raw.put("_HADOOP_CONFIG_DIR_", "/opt/hadoop/conf"); + + Map resolved = resolve(raw); + + Assertions.assertEquals("/tmp/jindo", resolved.get("fs.oss.tmp.data.dirs")); + Assertions.assertEquals("16", resolved.get("fs.oss.upload.thread.concurrency")); + Assertions.assertEquals("oss://bucket", resolved.get("fs.defaultFS")); + Assertions.assertEquals("hive", resolved.get("hadoop.username")); + // System-injected markers are not Hadoop config and must not leak through. + Assertions.assertFalse(resolved.containsKey("_STORAGE_TYPE_")); + Assertions.assertFalse(resolved.containsKey("_HADOOP_CONFIG_DIR_")); + // Derived keys still win over pass-through duplicates. + Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem", + resolved.get("fs.oss.impl")); + Assertions.assertEquals("ak", resolved.get("fs.oss.accessKeyId")); + } + + @Test + void convertedSecurityTokenRoundTrips() { + Map raw = new HashMap<>(); + raw.put("fs.oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + raw.put("fs.oss.accessKeyId", "ak"); + raw.put("fs.oss.accessKeySecret", "sk"); + raw.put("fs.oss.securityToken", "token"); + + Assertions.assertEquals("token", resolve(raw).get("fs.oss.securityToken")); + } + + @Test + void guessIsMeTrueForOssDlsEndpoint() { + Map raw = new HashMap<>(); + raw.put("oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + Assertions.assertTrue(OssHdfsProperties.guessIsMe(raw)); + } + + @Test + void guessIsMeFalseForPlainOssEndpoint() { + Map raw = new HashMap<>(); + raw.put("oss.endpoint", "oss-cn-beijing.aliyuncs.com"); + Assertions.assertFalse(OssHdfsProperties.guessIsMe(raw)); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystem.java b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystem.java index 08e81920078480..dd29d9b729ecf7 100644 --- a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystem.java @@ -25,6 +25,6 @@ public class OssFileSystem extends S3CompatibleFileSystem { public OssFileSystem(OssObjStorage objStorage) { - super(objStorage, objStorage.isUsePathStyle()); + super(objStorage, objStorage.isUsePathStyle(), objStorage.getSupportedSchemes()); } } diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProperties.java index 4687e88e7ba6d3..663b5596c242cd 100644 --- a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProperties.java @@ -36,6 +36,7 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -347,6 +348,11 @@ public String getUsePathStyle() { return usePathStyle; } + @Override + public Set getSupportedSchemes() { + return Set.of("oss", "s3", "s3a"); + } + public String getForceParsingByStandardUrl() { return forceParsingByStandardUrl; } diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProvider.java index 41cea83b088ba5..c4b3aaacd4809e 100644 --- a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProvider.java @@ -39,19 +39,42 @@ public class OssFileSystemProvider implements FileSystemProvider properties) { + // OSS-HDFS (JindoFS) is served by OssHdfsFileSystemProvider. Routing is first-match-wins over + // an unordered list, so native OSS must positively disclaim any OSS-HDFS config to stay disjoint. + if (STORAGE_TYPE_OSS_HDFS.equalsIgnoreCase(properties.get(STORAGE_TYPE_KEY))) { + return false; + } + if (STORAGE_TYPE_OSS.equalsIgnoreCase(properties.get(STORAGE_TYPE_KEY))) { + return true; + } + // Explicit OSS-HDFS flags outrank even fs.oss.support in the kernel's + // StorageProperties.createPrimary; mirror that precedence here. + if (Boolean.parseBoolean(properties.getOrDefault(FS_OSS_HDFS_SUPPORT, "false")) + || Boolean.parseBoolean(properties.getOrDefault(DEPRECATED_OSS_HDFS_SUPPORT, "false"))) { + return false; + } if (isExplicitOss(properties)) { return true; } String endpoint = firstPresent(properties, ENDPOINT_NAMES); - return endpoint != null && endpoint.contains("aliyuncs.com"); + if (endpoint == null || !endpoint.contains("aliyuncs.com")) { + return false; + } + return !endpoint.contains(OSS_HDFS_ENDPOINT_MARKER); } @Override @@ -80,8 +103,7 @@ public Set sensitivePropertyKeys() { } private boolean isExplicitOss(Map properties) { - return STORAGE_TYPE_OSS.equalsIgnoreCase(properties.get(STORAGE_TYPE_KEY)) - || STORAGE_TYPE_OSS.equalsIgnoreCase(properties.get(PROVIDER_KEY)) + return STORAGE_TYPE_OSS.equalsIgnoreCase(properties.get(PROVIDER_KEY)) || Boolean.parseBoolean(properties.getOrDefault(FS_OSS_SUPPORT, "false")); } diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssObjStorage.java b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssObjStorage.java index 65b52f9c6af10f..cfb1da6853270c 100644 --- a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssObjStorage.java @@ -69,6 +69,7 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -109,6 +110,11 @@ public boolean isUsePathStyle() { return properties.isUsePathStyle(); } + /** Returns the URI schemes this provider accepts (e.g. {@code {oss, s3, s3a}}). */ + public Set getSupportedSchemes() { + return properties.getSupportedSchemes(); + } + @Override public OSS getClient() throws IOException { return getClient(properties.getBucket()); @@ -193,7 +199,7 @@ public RemoteObjects listObjects(String remotePath, String continuationToken) th @Override public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions options) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); ListObjectsRequest request = new ListObjectsRequest(uri.bucket()); request.setPrefix(uri.key()); if (options != null) { @@ -214,8 +220,7 @@ public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions List objects = listing.getObjectSummaries().stream() .map(obj -> toRemoteObject(uri.key(), obj)) .collect(Collectors.toList()); - return new RemoteObjects(objects, listing.isTruncated(), - listing.isTruncated() ? listing.getNextMarker() : null); + return new RemoteObjects(objects, listing.isTruncated(), resolveNextMarker(listing)); } catch (OSSException e) { // OSSException (server-side errors such as NoSuchBucket) is a sibling of // ClientException, not a subclass, so it must be caught explicitly or it would @@ -228,7 +233,7 @@ public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions @Override public RemoteObject headObject(String remotePath) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { ObjectMetadata metadata = getClient(uri.bucket()).getObjectMetadata(uri.bucket(), uri.key()); return new RemoteObject(uri.key(), uri.key(), metadata.getETag(), metadata.getContentLength(), @@ -245,19 +250,21 @@ public RemoteObject headObject(String remotePath) throws IOException { @Override public void putObject(String remotePath, RequestBody requestBody) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(requestBody.contentLength()); try (InputStream content = requestBody.content()) { getClient(uri.bucket()).putObject(new PutObjectRequest(uri.bucket(), uri.key(), content, metadata)); - } catch (ClientException e) { + } catch (OSSException | ClientException e) { + // OSSException (server-side error) is a sibling of ClientException, not a subclass, + // so both must be caught or a server error would escape as an unchecked exception. throw new IOException("putObject failed for " + remotePath + ": " + e.getMessage(), e); } } @Override public void deleteObject(String remotePath) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { getClient(uri.bucket()).deleteObject(uri.bucket(), uri.key()); } catch (OSSException e) { @@ -272,12 +279,12 @@ public void deleteObject(String remotePath) throws IOException { @Override public void copyObject(String srcPath, String dstPath) throws IOException { - ObjectStorageUri src = ObjectStorageUri.parse(srcPath, false); - ObjectStorageUri dst = ObjectStorageUri.parse(dstPath, false); + ObjectStorageUri src = ObjectStorageUri.parse(srcPath, isUsePathStyle(), getSupportedSchemes()); + ObjectStorageUri dst = ObjectStorageUri.parse(dstPath, isUsePathStyle(), getSupportedSchemes()); try { getClient(src.bucket()).copyObject(new CopyObjectRequest( src.bucket(), src.key(), dst.bucket(), dst.key())); - } catch (ClientException e) { + } catch (OSSException | ClientException e) { throw new IOException("copyObject from " + srcPath + " to " + dstPath + " failed: " + e.getMessage(), e); } @@ -285,12 +292,12 @@ public void copyObject(String srcPath, String dstPath) throws IOException { @Override public String initiateMultipartUpload(String remotePath) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { InitiateMultipartUploadResult result = getClient(uri.bucket()).initiateMultipartUpload( new InitiateMultipartUploadRequest(uri.bucket(), uri.key())); return result.getUploadId(); - } catch (ClientException e) { + } catch (OSSException | ClientException e) { throw new IOException("initiateMultipartUpload failed for " + remotePath + ": " + e.getMessage(), e); } @@ -299,7 +306,7 @@ public String initiateMultipartUpload(String remotePath) throws IOException { @Override public UploadPartResult uploadPart(String remotePath, String uploadId, int partNum, RequestBody body) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try (InputStream content = body.content()) { com.aliyun.oss.model.UploadPartRequest request = new com.aliyun.oss.model.UploadPartRequest(); request.setBucketName(uri.bucket()); @@ -310,7 +317,7 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN request.setInputStream(content); com.aliyun.oss.model.UploadPartResult result = getClient(uri.bucket()).uploadPart(request); return new UploadPartResult(partNum, result.getETag()); - } catch (ClientException e) { + } catch (OSSException | ClientException e) { throw new IOException("uploadPart " + partNum + " failed for " + remotePath + ": " + e.getMessage(), e); } @@ -319,14 +326,14 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN @Override public void completeMultipartUpload(String remotePath, String uploadId, List parts) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); List partEtags = parts.stream() .map(part -> new PartETag(part.partNumber(), part.etag())) .collect(Collectors.toList()); try { getClient(uri.bucket()).completeMultipartUpload(new CompleteMultipartUploadRequest( uri.bucket(), uri.key(), uploadId, partEtags)); - } catch (ClientException e) { + } catch (OSSException | ClientException e) { throw new IOException("completeMultipartUpload failed for " + remotePath + ": " + e.getMessage(), e); } @@ -334,11 +341,11 @@ public void completeMultipartUpload(String remotePath, String uploadId, @Override public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { getClient(uri.bucket()).abortMultipartUpload(new AbortMultipartUploadRequest( uri.bucket(), uri.key(), uploadId)); - } catch (ClientException e) { + } catch (OSSException | ClientException e) { throw new IOException("abortMultipartUpload failed for " + remotePath + " (uploadId=" + uploadId + "): " + e.getMessage(), e); } @@ -346,7 +353,7 @@ public void abortMultipartUpload(String remotePath, String uploadId) throws IOEx @Override public InputStream openInputStreamAt(String remotePath, long fromByte) throws IOException { - ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, false); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, isUsePathStyle(), getSupportedSchemes()); try { GetObjectRequest request = new GetObjectRequest(uri.bucket(), uri.key()); if (fromByte > 0) { @@ -426,9 +433,9 @@ public String getPresignedUrl(String objectKey) throws IOException { GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket, objectKey, HttpMethod.PUT); request.setExpiration(expiration); URL signedUrl = getClient(bucket).generatePresignedUrl(request); - LOG.info("Generated OSS presigned URL for key={}", objectKey); + LOG.debug("Generated OSS presigned URL for key={}", objectKey); return signedUrl.toString(); - } catch (ClientException e) { + } catch (OSSException | ClientException e) { LOG.warn("Failed to generate OSS presigned URL for key={}", objectKey, e); throw new IOException("Failed to generate OSS presigned URL: " + e.getMessage(), e); } @@ -444,7 +451,7 @@ public void deleteObjectsByKeys(String bucket, List keys) throws IOExcep request.setKeys(batch); getClient(bucket).deleteObjects(request); } - } catch (ClientException e) { + } catch (OSSException | ClientException e) { throw new IOException("Failed to batch delete objects from bucket=" + bucket + ": " + e.getMessage(), e); } } @@ -473,6 +480,51 @@ private static String requireProperty(String value, String key, String descripti return value; } + /** + * Computes the marker that resumes listing on the next page. + * + *

OSS (like the S3 V1 {@code GetBucket} API) only fills + * {@link ObjectListing#getNextMarker()} when a {@code delimiter} is set. For a delimiter-less + * (recursive) listing the marker is {@code null} even when {@link ObjectListing#isTruncated()} + * is {@code true}, so naively forwarding it would stop pagination after the first page (~1000 + * keys) and silently drop every object beyond it (recursive delete/rename/list). + * + *

When the SDK leaves the marker blank on a truncated page, fall back to the + * lexicographically greatest key/common-prefix returned on this page — OSS lists in + * lexicographic order and resumes strictly after the marker. This mirrors the last-key + * fallback the COS/OBS SDKs perform internally. + * + *

If a page is marked truncated yet yields neither a marker nor any key/common-prefix to + * resume from, there is no cursor for the next page; returning {@code null} would make the + * paginating caller re-list from the start indefinitely, so this fails loudly instead. + */ + private static String resolveNextMarker(ObjectListing listing) throws IOException { + if (!listing.isTruncated()) { + return null; + } + String nextMarker = listing.getNextMarker(); + if (hasText(nextMarker)) { + return nextMarker; + } + String marker = null; + List summaries = listing.getObjectSummaries(); + if (summaries != null && !summaries.isEmpty()) { + marker = summaries.get(summaries.size() - 1).getKey(); + } + List commonPrefixes = listing.getCommonPrefixes(); + if (commonPrefixes != null && !commonPrefixes.isEmpty()) { + String lastPrefix = commonPrefixes.get(commonPrefixes.size() - 1); + if (marker == null || lastPrefix.compareTo(marker) > 0) { + marker = lastPrefix; + } + } + if (marker == null) { + throw new IOException("OSS reported a truncated listing but returned no marker and no " + + "keys to resume from; cannot paginate without looping from the start"); + } + return marker; + } + private static boolean isNotFound(OSSException e) { return "NoSuchKey".equals(e.getErrorCode()) || "NoSuchBucket".equals(e.getErrorCode()); diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssFileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssFileSystemProviderTest.java new file mode 100644 index 00000000000000..1da7916e4d5bac --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssFileSystemProviderTest.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.oss; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * Native-OSS routing must stay disjoint from OSS-HDFS (JindoFS). OSS-HDFS backend config carries + * {@code _STORAGE_TYPE_=OSS_HDFS} and an {@code fs.oss.endpoint} on {@code *.oss-dls.aliyuncs.com}; + * that endpoint contains {@code aliyuncs.com}, so the plain endpoint heuristic would otherwise make + * this native-OSS provider claim an OSS-HDFS config too. Since providers are picked first-match-wins + * over an unordered list, that overlap would make selection depend on registration order. + */ +class OssFileSystemProviderTest { + + private final OssFileSystemProvider provider = new OssFileSystemProvider(); + + @Test + void ossHdfsMarkerIsNotClaimedEvenWithAliyunEndpoint() { + Map props = new HashMap<>(); + props.put("_STORAGE_TYPE_", "OSS_HDFS"); + props.put("fs.oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + Assertions.assertFalse(provider.supports(props), + "native OSS must not claim an OSS-HDFS config: " + props); + } + + @Test + void ossDlsEndpointWithoutMarkerIsNotClaimed() { + Map props = new HashMap<>(); + props.put("fs.oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + Assertions.assertFalse(provider.supports(props), + "oss-dls endpoint belongs to OSS-HDFS, not native OSS: " + props); + } + + @Test + void nativeOssMarkerIsClaimed() { + Map props = new HashMap<>(); + props.put("_STORAGE_TYPE_", "OSS"); + Assertions.assertTrue(provider.supports(props)); + } + + @Test + void plainAliyunEndpointIsClaimed() { + Map props = new HashMap<>(); + props.put("oss.endpoint", "oss-cn-beijing.aliyuncs.com"); + Assertions.assertTrue(provider.supports(props)); + } + + @Test + void explicitOssHdfsFlagsAreNotClaimed() { + // StorageProperties.createPrimary treats these flags as OSS-HDFS before native OSS, + // even when the endpoint is a plain (non-dls) Aliyun endpoint. + Map props = new HashMap<>(); + props.put("fs.oss-hdfs.support", "true"); + props.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); + Assertions.assertFalse(provider.supports(props), + "explicit OSS-HDFS flag belongs to OssHdfsFileSystemProvider: " + props); + + Map deprecated = new HashMap<>(); + deprecated.put("oss.hdfs.enabled", "true"); + deprecated.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); + Assertions.assertFalse(provider.supports(deprecated), + "deprecated OSS-HDFS flag belongs to OssHdfsFileSystemProvider: " + deprecated); + } + + @Test + void ossHdfsFlagWinsOverNativeOssFlag() { + // Kernel precedence: OSS-HDFS flags are checked before fs.oss.support. + Map props = new HashMap<>(); + props.put("fs.oss-hdfs.support", "true"); + props.put("fs.oss.support", "true"); + Assertions.assertFalse(provider.supports(props)); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssObjStorageTest.java b/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssObjStorageTest.java index 8437be6f3efe59..4b71efda3c4e7f 100644 --- a/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssObjStorageTest.java +++ b/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssObjStorageTest.java @@ -206,6 +206,81 @@ void listObjects_usesOssNativeClientAndMapsResult() throws Exception { Assertions.assertEquals("marker-1", captor.getValue().getMarker()); } + @Test + void listObjects_truncatedWithoutNextMarker_fallsBackToLastKey() throws Exception { + // OSS only fills NextMarker when a delimiter is set; a delimiter-less (recursive) + // truncated page returns a null/blank marker. The fallback must resume from the last + // returned key, otherwise recursive list/delete/rename silently stop after page 1. + OSS mockOss = Mockito.mock(OSS.class); + OSSObjectSummary s1 = new OSSObjectSummary(); + s1.setKey("stage/a.parquet"); + OSSObjectSummary s2 = new OSSObjectSummary(); + s2.setKey("stage/b.parquet"); + ObjectListing listing = new ObjectListing(); + listing.setObjectSummaries(List.of(s1, s2)); + listing.setTruncated(true); + listing.setNextMarker(null); + Mockito.when(mockOss.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(listing); + + OssObjStorage storage = new TestableOssObjStorage(buildBasicProps(), mockOss); + RemoteObjects result = storage.listObjects("oss://my-bucket/stage/", null); + + Assertions.assertTrue(result.isTruncated()); + Assertions.assertEquals("stage/b.parquet", result.getContinuationToken(), + "Truncated page with blank NextMarker must fall back to the last object key"); + } + + @Test + void listObjects_truncatedButNoResolvableMarker_throwsInsteadOfLooping() { + // A truncated page with a blank NextMarker AND no keys/common-prefixes leaves no cursor to + // resume from. Returning a null marker with truncated=true would make the paginating caller + // re-list from the start forever. Fail loudly instead of silently looping. + OSS mockOss = Mockito.mock(OSS.class); + ObjectListing listing = new ObjectListing(); + listing.setObjectSummaries(Collections.emptyList()); + listing.setTruncated(true); + listing.setNextMarker(null); + Mockito.when(mockOss.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(listing); + + OssObjStorage storage = new TestableOssObjStorage(buildBasicProps(), mockOss); + + Assertions.assertThrows(IOException.class, + () -> storage.listObjects("oss://my-bucket/stage/", null), + "Truncated page with no resolvable marker must throw, not loop forever"); + } + + @Test + void listObjects_notTruncated_hasNoContinuationToken() throws Exception { + OSS mockOss = Mockito.mock(OSS.class); + OSSObjectSummary s1 = new OSSObjectSummary(); + s1.setKey("stage/a.parquet"); + ObjectListing listing = new ObjectListing(); + listing.setObjectSummaries(List.of(s1)); + listing.setTruncated(false); + Mockito.when(mockOss.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(listing); + + OssObjStorage storage = new TestableOssObjStorage(buildBasicProps(), mockOss); + RemoteObjects result = storage.listObjects("oss://my-bucket/stage/", null); + + Assertions.assertFalse(result.isTruncated()); + Assertions.assertNull(result.getContinuationToken()); + } + + @Test + void putObject_translatesServerSideOssExceptionToIoException() { + // OSSException (server-side, e.g. AccessDenied) is a sibling of ClientException, not a + // subclass; it must be caught and translated, not allowed to escape as an unchecked + // exception that bypasses the throws-IOException contract. + OSS mockOss = Mockito.mock(OSS.class); + Mockito.when(mockOss.putObject(Mockito.any(PutObjectRequest.class))) + .thenThrow(new com.aliyun.oss.OSSException("Access denied")); + OssObjStorage storage = new TestableOssObjStorage(buildBasicProps(), mockOss); + + byte[] content = "x".getBytes(StandardCharsets.UTF_8); + Assertions.assertThrows(IOException.class, () -> storage.putObject("oss://my-bucket/k.txt", + RequestBody.of(new ByteArrayInputStream(content), content.length))); + } + @Test void headObject_usesOssNativeClientAndMapsMetadata() throws Exception { OSS mockOss = Mockito.mock(OSS.class); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java index 0f6eeb35608d97..f738653ebec568 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java @@ -34,12 +34,12 @@ public S3FileSystem(S3FileSystemProperties properties) { } S3FileSystem(S3FileSystemProperties properties, S3ObjStorage objStorage) { - super(objStorage, objStorage.isUsePathStyle()); + super(objStorage, objStorage.isUsePathStyle(), objStorage.getSupportedSchemes()); this.properties = properties; } public S3FileSystem(S3ObjStorage objStorage) { - super(objStorage, objStorage.isUsePathStyle()); + super(objStorage, objStorage.isUsePathStyle(), objStorage.getSupportedSchemes()); this.properties = null; } diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java index 34e8fb54b934bd..97cfbb2c4b03d0 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java @@ -36,6 +36,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -213,6 +214,11 @@ public void validate() { .validate("Invalid S3 filesystem properties"); } + @Override + public Set getSupportedSchemes() { + return Set.of("s3", "s3a", "s3n"); + } + @Override public String providerName() { return "S3"; diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java index 274facf16529c7..6ddf0434966023 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java @@ -20,6 +20,7 @@ import org.apache.doris.filesystem.UploadPartResult; import org.apache.doris.filesystem.spi.ObjStorage; import org.apache.doris.filesystem.spi.ObjectListOptions; +import org.apache.doris.filesystem.spi.ObjectStorageUri; import org.apache.doris.filesystem.spi.RemoteObject; import org.apache.doris.filesystem.spi.RemoteObjects; import org.apache.doris.filesystem.spi.StsCredentials; @@ -77,6 +78,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -119,6 +121,11 @@ public boolean isUsePathStyle() { return usePathStyle; } + /** Returns the URI schemes this provider accepts (e.g. {@code {s3, s3a, s3n}}). */ + public Set getSupportedSchemes() { + return s3Properties.getSupportedSchemes(); + } + @Override public S3Client getClient() throws IOException { if (closed.get()) { @@ -202,7 +209,7 @@ public RemoteObjects listObjects(String remotePath, String continuationToken) th @Override public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions options) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder() .bucket(uri.bucket()) .prefix(uri.key()); @@ -283,7 +290,7 @@ public RemoteObjects listObjectsNonRecursive(String remotePath, String continuat @Override public org.apache.doris.filesystem.spi.RemoteObject headObject(String remotePath) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try { HeadObjectResponse response = getClient().headObject( HeadObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); @@ -305,7 +312,7 @@ public org.apache.doris.filesystem.spi.RemoteObject headObject(String remotePath @Override public void putObject(String remotePath, org.apache.doris.filesystem.spi.RequestBody requestBody) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try (InputStream content = requestBody.content()) { getClient().putObject( PutObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build(), @@ -318,7 +325,7 @@ public void putObject(String remotePath, org.apache.doris.filesystem.spi.Request @Override public void deleteObject(String remotePath) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try { getClient().deleteObject(DeleteObjectRequest.builder() .bucket(uri.bucket()).key(uri.key()).build()); @@ -334,8 +341,8 @@ public void deleteObject(String remotePath) throws IOException { @Override public void copyObject(String srcPath, String dstPath) throws IOException { - S3Uri srcUri = S3Uri.parse(srcPath, usePathStyle); - S3Uri dstUri = S3Uri.parse(dstPath, usePathStyle); + ObjectStorageUri srcUri = ObjectStorageUri.parse(srcPath, usePathStyle, getSupportedSchemes()); + ObjectStorageUri dstUri = ObjectStorageUri.parse(dstPath, usePathStyle, getSupportedSchemes()); try { getClient().copyObject(CopyObjectRequest.builder() .copySource(SdkHttpUtils.urlEncodeIgnoreSlashes( @@ -351,7 +358,7 @@ public void copyObject(String srcPath, String dstPath) throws IOException { @Override public String initiateMultipartUpload(String remotePath) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try { CreateMultipartUploadResponse response = getClient().createMultipartUpload( CreateMultipartUploadRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); @@ -365,7 +372,7 @@ public String initiateMultipartUpload(String remotePath) throws IOException { @Override public UploadPartResult uploadPart(String remotePath, String uploadId, int partNum, org.apache.doris.filesystem.spi.RequestBody body) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try (InputStream content = body.content()) { UploadPartResponse response = getClient().uploadPart( UploadPartRequest.builder() @@ -385,7 +392,7 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN @Override public void completeMultipartUpload(String remotePath, String uploadId, List parts) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); List completedParts = parts.stream() .map(p -> CompletedPart.builder().partNumber(p.partNumber()).eTag(p.etag()).build()) .collect(Collectors.toList()); @@ -402,7 +409,7 @@ public void completeMultipartUpload(String remotePath, String uploadId, @Override public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try { getClient().abortMultipartUpload(AbortMultipartUploadRequest.builder() .bucket(uri.bucket()).key(uri.key()).uploadId(uploadId).build()); @@ -422,7 +429,7 @@ public void abortMultipartUpload(String remotePath, String uploadId) throws IOEx * Open an InputStream for reading an S3 object. */ InputStream openInputStream(String remotePath) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try { return getClient().getObject( GetObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); @@ -438,7 +445,7 @@ InputStream openInputStream(String remotePath) throws IOException { */ @Override public InputStream openInputStreamAt(String remotePath, long fromByte) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try { GetObjectRequest.Builder req = GetObjectRequest.builder() .bucket(uri.bucket()).key(uri.key()); @@ -458,7 +465,7 @@ public InputStream openInputStreamAt(String remotePath, long fromByte) throws IO */ @Override public long headObjectLastModified(String remotePath) throws IOException { - S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try { HeadObjectResponse resp = getClient().headObject( HeadObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3Uri.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3Uri.java deleted file mode 100644 index 4d8614739f6f86..00000000000000 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3Uri.java +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.filesystem.s3; - -/** - * Parsed S3/S3A URI. Extracts bucket and key without any fe-core dependency. - */ -public final class S3Uri { - - private final String bucket; - private final String key; - - private S3Uri(String bucket, String key) { - this.bucket = bucket; - this.key = key; - } - - /** - * Parses URIs of the form: s3://bucket/key, s3a://bucket/key, s3n://bucket/key. - * - *

When {@code pathStyleAccess} is {@code true} and the scheme is - * {@code http} or {@code https}, the URI is treated as path-style: - * {@code scheme://endpoint/bucket/key}. The first path segment after the host is the - * bucket, the remainder is the key. For all other schemes (including {@code s3://}, - * {@code s3a://}, {@code s3n://}) the first authority component is always treated as - * the bucket, regardless of {@code pathStyleAccess}. - */ - public static S3Uri parse(String path, boolean pathStyleAccess) { - if (path == null) { - throw new IllegalArgumentException("S3 path must not be null"); - } - int schemeEnd = path.indexOf("://"); - if (schemeEnd < 0) { - throw new IllegalArgumentException("Cannot parse S3 URI without scheme: " + path); - } - String scheme = path.substring(0, schemeEnd); - String withoutScheme = path.substring(schemeEnd + 3); - - boolean httpScheme = "http".equalsIgnoreCase(scheme) || "https".equalsIgnoreCase(scheme); - if (pathStyleAccess && httpScheme) { - // Path-style: scheme://host/bucket/key - int hostEnd = withoutScheme.indexOf('/'); - if (hostEnd < 0) { - // scheme://host with no bucket - throw new IllegalArgumentException( - "Path-style URI is missing bucket segment: " + path); - } - String afterHost = withoutScheme.substring(hostEnd + 1); - // Strip extra leading slashes between host and bucket. - while (afterHost.startsWith("/")) { - afterHost = afterHost.substring(1); - } - int bucketEnd = afterHost.indexOf('/'); - if (bucketEnd < 0) { - if (afterHost.isEmpty()) { - throw new IllegalArgumentException( - "Path-style URI is missing bucket segment: " + path); - } - return new S3Uri(afterHost, ""); - } - String rawKey = afterHost.substring(bucketEnd + 1); - while (rawKey.startsWith("/")) { - rawKey = rawKey.substring(1); - } - return new S3Uri(afterHost.substring(0, bucketEnd), rawKey); - } - - // Virtual-hosted style: scheme://bucket/key - int slashIdx = withoutScheme.indexOf('/'); - if (slashIdx < 0) { - return new S3Uri(withoutScheme, ""); - } - String rawKey = withoutScheme.substring(slashIdx + 1); - // Strip leading slashes to normalize keys (e.g., "s3://bucket//path" → key "path") - while (rawKey.startsWith("/")) { - rawKey = rawKey.substring(1); - } - return new S3Uri(withoutScheme.substring(0, slashIdx), rawKey); - } - - public String bucket() { - return bucket; - } - - public String key() { - return key; - } -} diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3UriTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3UriTest.java deleted file mode 100644 index 0e655f8bf66d6a..00000000000000 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3UriTest.java +++ /dev/null @@ -1,133 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.filesystem.s3; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class S3UriTest { - - @Test - void parseS3VirtualHostedStyle() { - S3Uri uri = S3Uri.parse("s3://my-bucket/path/to/file.csv", false); - Assertions.assertEquals("my-bucket", uri.bucket()); - Assertions.assertEquals("path/to/file.csv", uri.key()); - } - - @Test - void parseS3aScheme() { - S3Uri uri = S3Uri.parse("s3a://my-bucket/key", false); - Assertions.assertEquals("my-bucket", uri.bucket()); - Assertions.assertEquals("key", uri.key()); - } - - @Test - void parseS3nScheme() { - S3Uri uri = S3Uri.parse("s3n://my-bucket/dir/file", false); - Assertions.assertEquals("my-bucket", uri.bucket()); - Assertions.assertEquals("dir/file", uri.key()); - } - - @Test - void parseBucketOnly() { - S3Uri uri = S3Uri.parse("s3://my-bucket", false); - Assertions.assertEquals("my-bucket", uri.bucket()); - Assertions.assertEquals("", uri.key()); - } - - @Test - void parseNormalizesDoubleSlashInKey() { - S3Uri uri = S3Uri.parse("s3://bucket//path", false); - Assertions.assertEquals("bucket", uri.bucket()); - Assertions.assertEquals("path", uri.key()); - } - - @Test - void parseHttpsEndpoint() { - S3Uri uri = S3Uri.parse("https://s3.amazonaws.com/my-bucket/key", false); - Assertions.assertEquals("s3.amazonaws.com", uri.bucket()); - Assertions.assertEquals("my-bucket/key", uri.key()); - } - - @Test - void nullPathThrows() { - Assertions.assertThrows(IllegalArgumentException.class, () -> S3Uri.parse(null, false)); - } - - @Test - void noSchemeThrows() { - Assertions.assertThrows(IllegalArgumentException.class, () -> S3Uri.parse("bucket/key", false)); - } - - // ------------------------------------------------------------------ - // Path-style HTTP(S) parsing (#9) - // ------------------------------------------------------------------ - - @Test - void parsePathStyleHttpsTreatsFirstPathSegmentAsBucket() { - S3Uri uri = S3Uri.parse("https://endpoint.example.com/my-bucket/key/x.csv", true); - Assertions.assertEquals("my-bucket", uri.bucket()); - Assertions.assertEquals("key/x.csv", uri.key()); - } - - @Test - void parsePathStyleHttpAlsoSupported() { - S3Uri uri = S3Uri.parse("http://10.0.0.1:9000/data/dir/file", true); - Assertions.assertEquals("data", uri.bucket()); - Assertions.assertEquals("dir/file", uri.key()); - } - - @Test - void parsePathStyleBucketOnly() { - S3Uri uri = S3Uri.parse("https://endpoint/my-bucket", true); - Assertions.assertEquals("my-bucket", uri.bucket()); - Assertions.assertEquals("", uri.key()); - } - - @Test - void parsePathStyleTrailingSlashIsEmptyKey() { - S3Uri uri = S3Uri.parse("https://endpoint/my-bucket/", true); - Assertions.assertEquals("my-bucket", uri.bucket()); - Assertions.assertEquals("", uri.key()); - } - - @Test - void parsePathStyleMissingBucketThrows() { - Assertions.assertThrows(IllegalArgumentException.class, - () -> S3Uri.parse("https://endpoint", true)); - Assertions.assertThrows(IllegalArgumentException.class, - () -> S3Uri.parse("https://endpoint/", true)); - } - - @Test - void parseS3SchemeIgnoresPathStyleFlag() { - // pathStyleAccess only affects http/https URIs; s3:// is always virtual-hosted. - S3Uri uri = S3Uri.parse("s3://my-bucket/path/to/file", true); - Assertions.assertEquals("my-bucket", uri.bucket()); - Assertions.assertEquals("path/to/file", uri.key()); - } - - @Test - void parseHttpsWithoutPathStyleKeepsHostAsBucket() { - // Backwards compatibility: when pathStyleAccess=false, https://endpoint/bucket/key - // is parsed with the host as the "bucket" (existing behaviour preserved). - S3Uri uri = S3Uri.parse("https://s3.amazonaws.com/my-bucket/key", false); - Assertions.assertEquals("s3.amazonaws.com", uri.bucket()); - Assertions.assertEquals("my-bucket/key", uri.key()); - } -} diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectStorageUri.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectStorageUri.java index eaf6e818361fd0..ef9fbd8c4701e6 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectStorageUri.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectStorageUri.java @@ -17,21 +17,28 @@ package org.apache.doris.filesystem.spi; +import java.util.Set; + /** - * Parsed object-storage URI. Extracts bucket and key without any fe-core dependency. + * Parsed object-storage URI (bucket + key). Has no fe-core dependency. + * + *

This is the single shared parser for every S3-compatible provider (S3, OSS, COS, OBS). + * The bucket/key extraction mechanics are genuinely uniform across them; the per-provider + * variation is supplied by the caller: the {@code pathStyleAccess} flag and the set of accepted + * schemes. The provider itself is selected from connection properties (endpoint domain / explicit + * storage type), not from the URI scheme, but each provider still constrains which schemes it will + * accept via {@link #parse(String, boolean, Set)} — so COS accepts {@code s3}/{@code s3a}/{@code + * cos}/{@code cosn} but rejects {@code oss}. Each provider owns its accepted-scheme set on its + * {@code S3CompatibleFileSystemProperties}; this class only enforces it. * - *

FIXME: this should be an interface, with each provider implementing its own - * {@code normalize}. The current single shared parser is essentially the legacy approach: - * historically OBS/COS/OSS/S3 URIs all had to be normalized into S3 form, either to hand - * to the BE or for the FE's own use. With per-provider native SDKs that constraint no - * longer holds. {@code pathStyleAccess} is part of the same legacy and should not appear - * here either — it is an S3-family endpoint concern that belongs in the provider - * implementations, not in generic URI parsing. + *

{@code pathStyleAccess} stays here because parsing a path-style HTTP(S) endpoint URL + * genuinely needs it to locate the bucket segment; it is independent of whether a provider's + * client performs path-style addressing (e.g. COS parses path-style URLs but its SDK + * always issues virtual-hosted requests). * - *

FIXME: the input is a plain {@code String} that is then split on {@code '/'} below, - * so URI escaping must already have been applied before a value can be used as input here - * (see the note on {@link #parse}). An interface-based design should take a properly - * typed/encoded URI instead of a raw string. + *

Encoding: the input is a plain {@code String} split on {@code '/'} (see + * {@link #parse}); URI escaping must already have been applied by the caller. Any {@code /} + * inside a bucket or key must be percent-encoded, otherwise it is mistaken for a separator. */ public final class ObjectStorageUri { @@ -59,6 +66,45 @@ private ObjectStorageUri(String bucket, String key) { * original (escaped) form. */ public static ObjectStorageUri parse(String path, boolean pathStyleAccess) { + return parse(path, pathStyleAccess, null); + } + + /** + * Same as {@link #parse(String, boolean)} but validates the URI scheme against the set of + * schemes the calling provider accepts. A scheme not in {@code supportedSchemes} (matched + * case-insensitively) is rejected, so e.g. a COS provider refuses an {@code oss://} URI. + * Passing {@code null} or an empty set skips scheme validation. + * + *

{@code http}/{@code https} are always accepted regardless of {@code supportedSchemes}: + * they are the endpoint-URL form (e.g. a TVF such as {@code s3()} pointed at a path-style + * endpoint), not a provider-identity scheme. {@code supportedSchemes} therefore stays explicit + * (e.g. {@code s3}/{@code oss}/{@code cos}) for catalog scheme-to-storage routing and omits + * {@code http}/{@code https} on purpose. + */ + public static ObjectStorageUri parse(String path, boolean pathStyleAccess, Set supportedSchemes) { + if (path == null) { + throw new IllegalArgumentException("Object storage path must not be null"); + } + if (supportedSchemes != null && !supportedSchemes.isEmpty()) { + int schemeEnd = path.indexOf("://"); + if (schemeEnd < 0) { + throw new IllegalArgumentException("Cannot parse object storage URI without scheme: " + path); + } + String scheme = path.substring(0, schemeEnd).toLowerCase(); + // http/https is the endpoint-URL form (e.g. TVF s3() with a path-style endpoint), + // not a provider-identity scheme, so it bypasses the supportedSchemes check. The set + // stays explicit (s3/oss/cos/...) for catalog scheme-to-storage routing; the actual + // bucket/key extraction for http(s) is handled by the path-style branch in doParse. + boolean httpEndpoint = scheme.equals("http") || scheme.equals("https"); + if (!httpEndpoint && !supportedSchemes.contains(scheme)) { + throw new IllegalArgumentException("Unsupported scheme '" + scheme + + "' for this storage; supported schemes: " + supportedSchemes); + } + } + return doParse(path, pathStyleAccess); + } + + private static ObjectStorageUri doParse(String path, boolean pathStyleAccess) { if (path == null) { throw new IllegalArgumentException("Object storage path must not be null"); } diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java index 5746fed5dd8549..5faee2bce6960d 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java @@ -37,6 +37,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Pattern; /** @@ -59,10 +60,13 @@ public abstract class S3CompatibleFileSystem extends ObjFileSystem { private static final Comparator UTF8_BINARY_ORDER = S3CompatibleFileSystem::compareUtf8Binary; private final boolean usePathStyle; + private final Set supportedSchemes; - protected S3CompatibleFileSystem(ObjStorage objStorage, boolean usePathStyle) { + protected S3CompatibleFileSystem(ObjStorage objStorage, boolean usePathStyle, + Set supportedSchemes) { super(objStorage); this.usePathStyle = usePathStyle; + this.supportedSchemes = supportedSchemes; } @Override @@ -208,7 +212,7 @@ private void deleteRecursive(String prefix) throws IOException { /** Parses {@code uri} respecting the underlying client's path-style configuration. */ private ObjectStorageUri parseUri(String uri) { - return ObjectStorageUri.parse(uri, usePathStyle); + return ObjectStorageUri.parse(uri, usePathStyle, supportedSchemes); } /** @@ -555,13 +559,11 @@ private class ObjectStorageFileIterator implements FileIterator { @Override public boolean hasNext() throws IOException { - if (bufferIdx < buffer.size()) { - return true; - } - if (done) { - return false; + // A truncated page may consist entirely of directory markers and filter down to an + // empty buffer; keep paging until a visible entry shows up or the listing is exhausted. + while (bufferIdx >= buffer.size() && !done) { + fetchNextPage(); } - fetchNextPage(); return bufferIdx < buffer.size(); } diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/ObjectStorageUriTest.java b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/ObjectStorageUriTest.java new file mode 100644 index 00000000000000..beb3289ff0df3c --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/ObjectStorageUriTest.java @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.spi; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +/** + * Unit tests for the single shared object-storage URI parser used by every S3-compatible + * provider (S3, OSS, COS, OBS). + */ +class ObjectStorageUriTest { + + @Test + void parseVirtualHostedStyle() { + ObjectStorageUri uri = ObjectStorageUri.parse("s3://my-bucket/path/to/file.csv", false); + Assertions.assertEquals("my-bucket", uri.bucket()); + Assertions.assertEquals("path/to/file.csv", uri.key()); + } + + @Test + void parseS3aScheme() { + ObjectStorageUri uri = ObjectStorageUri.parse("s3a://my-bucket/key", false); + Assertions.assertEquals("my-bucket", uri.bucket()); + Assertions.assertEquals("key", uri.key()); + } + + @Test + void parseS3nScheme() { + ObjectStorageUri uri = ObjectStorageUri.parse("s3n://my-bucket/dir/file", false); + Assertions.assertEquals("my-bucket", uri.bucket()); + Assertions.assertEquals("dir/file", uri.key()); + } + + @Test + void parseBucketOnly() { + ObjectStorageUri uri = ObjectStorageUri.parse("s3://my-bucket", false); + Assertions.assertEquals("my-bucket", uri.bucket()); + Assertions.assertEquals("", uri.key()); + } + + @Test + void parseNormalizesDoubleSlashInKey() { + ObjectStorageUri uri = ObjectStorageUri.parse("s3://bucket//path", false); + Assertions.assertEquals("bucket", uri.bucket()); + Assertions.assertEquals("path", uri.key()); + } + + @Test + void parseHttpsEndpoint() { + ObjectStorageUri uri = ObjectStorageUri.parse("https://s3.amazonaws.com/my-bucket/key", false); + Assertions.assertEquals("s3.amazonaws.com", uri.bucket()); + Assertions.assertEquals("my-bucket/key", uri.key()); + } + + @Test + void nullPathThrows() { + Assertions.assertThrows(IllegalArgumentException.class, () -> ObjectStorageUri.parse(null, false)); + } + + @Test + void noSchemeThrows() { + Assertions.assertThrows(IllegalArgumentException.class, () -> ObjectStorageUri.parse("bucket/key", false)); + } + + // ------------------------------------------------------------------ + // Scheme is not a discriminator: the provider is chosen by properties + // (endpoint domain / explicit storage type), not by URI scheme. The parser is + // intentionally lenient so that any provider's native scheme AND the historically + // normalized s3:// form parse the same way. e.g. a path supplied to the OSS/COS/OBS + // backend is frequently "s3://...". + // ------------------------------------------------------------------ + + @Test + void parseAcceptsCloudNativeSchemes() { + Assertions.assertEquals("b", ObjectStorageUri.parse("oss://b/k", false).bucket()); + Assertions.assertEquals("b", ObjectStorageUri.parse("cos://b/k", false).bucket()); + Assertions.assertEquals("b", ObjectStorageUri.parse("cosn://b/k", false).bucket()); + Assertions.assertEquals("b", ObjectStorageUri.parse("obs://b/k", false).bucket()); + Assertions.assertEquals("k", ObjectStorageUri.parse("cosn://b/k", false).key()); + } + + @Test + void parseAcceptsS3SchemeForCloudBackends() { + // The legacy normalized form: an s3:// path that actually targets an OSS/COS/OBS + // bucket must parse identically to the native-scheme form. + ObjectStorageUri uri = ObjectStorageUri.parse("s3://oss-bucket/dir/file", false); + Assertions.assertEquals("oss-bucket", uri.bucket()); + Assertions.assertEquals("dir/file", uri.key()); + } + + // ------------------------------------------------------------------ + // Path-style HTTP(S) parsing + // ------------------------------------------------------------------ + + @Test + void parsePathStyleHttpsTreatsFirstPathSegmentAsBucket() { + ObjectStorageUri uri = ObjectStorageUri.parse("https://endpoint.example.com/my-bucket/key/x.csv", true); + Assertions.assertEquals("my-bucket", uri.bucket()); + Assertions.assertEquals("key/x.csv", uri.key()); + } + + @Test + void parsePathStyleHttpAlsoSupported() { + ObjectStorageUri uri = ObjectStorageUri.parse("http://10.0.0.1:9000/data/dir/file", true); + Assertions.assertEquals("data", uri.bucket()); + Assertions.assertEquals("dir/file", uri.key()); + } + + @Test + void parsePathStyleBucketOnly() { + ObjectStorageUri uri = ObjectStorageUri.parse("https://endpoint/my-bucket", true); + Assertions.assertEquals("my-bucket", uri.bucket()); + Assertions.assertEquals("", uri.key()); + } + + @Test + void parsePathStyleTrailingSlashIsEmptyKey() { + ObjectStorageUri uri = ObjectStorageUri.parse("https://endpoint/my-bucket/", true); + Assertions.assertEquals("my-bucket", uri.bucket()); + Assertions.assertEquals("", uri.key()); + } + + @Test + void parsePathStyleMissingBucketThrows() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> ObjectStorageUri.parse("https://endpoint", true)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> ObjectStorageUri.parse("https://endpoint/", true)); + } + + @Test + void parseS3SchemeIgnoresPathStyleFlag() { + // pathStyleAccess only affects http/https URIs; s3:// (and other non-http schemes) are + // always virtual-hosted regardless of the flag. + ObjectStorageUri uri = ObjectStorageUri.parse("s3://my-bucket/path/to/file", true); + Assertions.assertEquals("my-bucket", uri.bucket()); + Assertions.assertEquals("path/to/file", uri.key()); + } + + // ------------------------------------------------------------------ + // Per-provider scheme validation (the parse(path, pathStyle, supportedSchemes) overload) + // ------------------------------------------------------------------ + + @Test + void parseAcceptsSchemeInSupportedSet() { + ObjectStorageUri uri = ObjectStorageUri.parse("s3://b/k", false, Set.of("s3", "s3a", "oss")); + Assertions.assertEquals("b", uri.bucket()); + Assertions.assertEquals("k", uri.key()); + } + + @Test + void parseRejectsForeignScheme() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> ObjectStorageUri.parse("oss://b/k", false, Set.of("cos", "cosn", "s3", "s3a"))); + } + + @Test + void parseAcceptsHttpsEndpointEvenWhenNotInSupportedSet() { + // http/https is the endpoint-URL form (used by TVF like s3()), not a provider-identity + // scheme, so it is accepted regardless of supportedSchemes (which stays explicit for + // catalog scheme-to-storage routing and intentionally omits http/https). Path-style on: + // first path segment is the bucket. + ObjectStorageUri uri = ObjectStorageUri.parse( + "https://endpoint.example.com/my-bucket/dir/file", true, Set.of("oss", "s3", "s3a")); + Assertions.assertEquals("my-bucket", uri.bucket()); + Assertions.assertEquals("dir/file", uri.key()); + } + + @Test + void parseAcceptsHttpEndpointEvenWhenNotInSupportedSet() { + ObjectStorageUri uri = ObjectStorageUri.parse( + "http://10.0.0.1:9000/data/dir/file", true, Set.of("cos", "cosn", "s3", "s3a")); + Assertions.assertEquals("data", uri.bucket()); + Assertions.assertEquals("dir/file", uri.key()); + } + + @Test + void parseHttpsWithoutPathStyleKeepsHostAsBucket() { + // When pathStyleAccess=false, https://endpoint/bucket/key parses the host as the + // "bucket" (existing behaviour preserved). + ObjectStorageUri uri = ObjectStorageUri.parse("https://s3.amazonaws.com/my-bucket/key", false); + Assertions.assertEquals("s3.amazonaws.com", uri.bucket()); + Assertions.assertEquals("my-bucket/key", uri.key()); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystemTest.java b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystemTest.java new file mode 100644 index 00000000000000..3a2d2f4c8d4332 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystemTest.java @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.spi; + +import org.apache.doris.filesystem.FileEntry; +import org.apache.doris.filesystem.FileIterator; +import org.apache.doris.filesystem.Location; +import org.apache.doris.filesystem.UploadPartResult; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Tests for {@link S3CompatibleFileSystem}'s paginating list iterator. + * + *

Object stores return directory-marker placeholders (keys ending in "/") interleaved with + * real objects, and a truncated page may contain nothing but markers. The iterator must keep + * paging past such fully-filtered pages instead of reporting end-of-listing. + */ +class S3CompatibleFileSystemTest { + + /** + * A truncated page whose keys are all directory markers filters down to an empty buffer; + * the iterator must continue to the next page rather than stop with files left behind. + */ + @Test + void testIteratorSkipsMarkerOnlyTruncatedPage() throws IOException { + PagingObjStorage storage = new PagingObjStorage(List.of( + new RemoteObjects(List.of( + remoteObject("data/a/"), + remoteObject("data/b/")), true, "t1"), + new RemoteObjects(List.of( + remoteObject("data/c/"), + remoteObject("data/file1.csv")), true, "t2"), + new RemoteObjects(List.of( + remoteObject("data/file2.csv")), false, null))); + TestFileSystem fs = new TestFileSystem(storage); + + List uris = new ArrayList<>(); + try (FileIterator it = fs.list(Location.of("s3://bucket/data"))) { + while (it.hasNext()) { + FileEntry entry = it.next(); + uris.add(entry.location().uri()); + } + } + + Assertions.assertEquals( + List.of("s3://bucket/data/file1.csv", "s3://bucket/data/file2.csv"), uris); + Assertions.assertEquals(3, storage.pagesServed); + } + + /** + * A marker-only final page must simply end the iteration, not loop or throw. + */ + @Test + void testIteratorEndsOnMarkerOnlyFinalPage() throws IOException { + PagingObjStorage storage = new PagingObjStorage(List.of( + new RemoteObjects(List.of( + remoteObject("data/file1.csv")), true, "t1"), + new RemoteObjects(List.of( + remoteObject("data/z/")), false, null))); + TestFileSystem fs = new TestFileSystem(storage); + + List uris = new ArrayList<>(); + try (FileIterator it = fs.list(Location.of("s3://bucket/data"))) { + while (it.hasNext()) { + uris.add(it.next().location().uri()); + } + } + + Assertions.assertEquals(List.of("s3://bucket/data/file1.csv"), uris); + Assertions.assertEquals(2, storage.pagesServed); + } + + private static RemoteObject remoteObject(String key) { + return new RemoteObject(key, key, "etag", key.endsWith("/") ? 0 : 10, 1234567890L); + } + + /** Minimal concrete subclass; only the inherited list iterator is exercised. */ + private static class TestFileSystem extends S3CompatibleFileSystem { + TestFileSystem(ObjStorage storage) { + super(storage, false, Set.of("s3")); + } + } + + /** + * {@link ObjStorage} stub serving a fixed sequence of listing pages, verifying that the + * continuation token of page N is echoed back when fetching page N + 1. + */ + private static class PagingObjStorage implements ObjStorage { + private final List pages; + private int pagesServed = 0; + + PagingObjStorage(List pages) { + this.pages = pages; + } + + @Override + public RemoteObjects listObjects(String remotePath, String continuationToken) throws IOException { + String expectedToken = pagesServed == 0 ? null : pages.get(pagesServed - 1).getContinuationToken(); + Assertions.assertEquals(expectedToken, continuationToken, + "listObjects must resume from the previous page's continuation token"); + Assertions.assertTrue(pagesServed < pages.size(), "listed past the final page"); + return pages.get(pagesServed++); + } + + @Override + public Object getClient() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public RemoteObject headObject(String remotePath) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void putObject(String remotePath, RequestBody requestBody) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteObject(String remotePath) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void copyObject(String srcPath, String dstPath) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public String initiateMultipartUpload(String remotePath) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public UploadPartResult uploadPart(String remotePath, String uploadId, + int partNum, RequestBody body) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void completeMultipartUpload(String remotePath, String uploadId, + List parts) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws IOException { + // no-op + } + } +} diff --git a/fe/fe-filesystem/pom.xml b/fe/fe-filesystem/pom.xml index 7333615dbd5639..14b1e5d479a1e6 100644 --- a/fe/fe-filesystem/pom.xml +++ b/fe/fe-filesystem/pom.xml @@ -52,7 +52,10 @@ under the License. fe-filesystem-obs fe-filesystem-local fe-filesystem-azure + fe-filesystem-hdfs-base fe-filesystem-hdfs + fe-filesystem-oss-hdfs + fe-filesystem-jfs fe-filesystem-broker fe-filesystem-http diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index cd2391b1775f1e..b300b6185c8730 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -771,11 +771,11 @@ addRollupClause ; alterTableClause - : ADD COLUMN columnDef columnPosition? toRollup? properties=propertyClause? #addColumnClause + : ADD COLUMN columnDefWithPath columnPosition? toRollup? properties=propertyClause? #addColumnClause | ADD COLUMN LEFT_PAREN columnDefs RIGHT_PAREN toRollup? properties=propertyClause? #addColumnsClause - | DROP COLUMN name=identifier fromRollup? properties=propertyClause? #dropColumnClause - | MODIFY COLUMN columnDef columnPosition? fromRollup? + | DROP COLUMN name=qualifiedName fromRollup? properties=propertyClause? #dropColumnClause + | MODIFY COLUMN columnDefWithPath columnPosition? fromRollup? properties=propertyClause? #modifyColumnClause | ORDER BY identifierList fromRollup? properties=propertyClause? #reorderColumnsClause | ADD TEMPORARY? partitionDef @@ -794,14 +794,14 @@ alterTableClause | RENAME newName=identifier #renameClause | RENAME ROLLUP name=identifier newName=identifier #renameRollupClause | RENAME PARTITION name=identifier newName=identifier #renamePartitionClause - | RENAME COLUMN name=identifier newName=identifier #renameColumnClause + | RENAME COLUMN name=qualifiedName TO? newName=identifier #renameColumnClause | ADD indexDef #addIndexClause | DROP INDEX (IF EXISTS)? name=identifier partitionSpec? #dropIndexClause | ENABLE FEATURE name=STRING_LITERAL (WITH properties=propertyClause)? #enableFeatureClause | MODIFY DISTRIBUTION (DISTRIBUTED BY (HASH hashKeys=identifierList | RANDOM) (BUCKETS (INTEGER_VALUE | autoBucket=AUTO))?)? #modifyDistributionClause | MODIFY COMMENT comment=STRING_LITERAL #modifyTableCommentClause - | MODIFY COLUMN name=identifier COMMENT comment=STRING_LITERAL #modifyColumnCommentClause + | MODIFY COLUMN name=qualifiedName COMMENT comment=STRING_LITERAL #modifyColumnCommentClause | MODIFY ENGINE TO name=identifier properties=propertyClause? #modifyEngineClause | ADD TEMPORARY? PARTITIONS FROM from=partitionValueList TO to=partitionValueList @@ -1554,6 +1554,17 @@ columnDef (COMMENT comment=STRING_LITERAL)? ; +columnDefWithPath + : columnDef + | colNames+=identifier (DOT colNames+=identifier)+ type=dataType + KEY? + (aggType=aggTypeDef)? + ((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)? + ((NOT)? nullable=NULL)? + (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? + (COMMENT comment=STRING_LITERAL)? + ; + indexDefs : indexes+=indexDef (COMMA indexes+=indexDef)* ; diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/StructField.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/StructField.java index ea8124c8aeb234..e9432c1efad1c5 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/StructField.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/StructField.java @@ -40,13 +40,22 @@ public class StructField { @SerializedName(value = "containsNull") private final boolean containsNull; // Now always true (nullable field) + // Runtime-only schema change intent; do not persist it as part of the table schema. + private transient boolean commentSpecified; + public static final String DEFAULT_FIELD_NAME = "col"; public StructField(String name, Type type, String comment, boolean containsNull) { + this(name, type, comment, containsNull, !Strings.isNullOrEmpty(comment)); + } + + public StructField(String name, Type type, String comment, boolean containsNull, + boolean commentSpecified) { this.name = name.toLowerCase(); this.type = type; this.comment = comment; this.containsNull = containsNull; + this.commentSpecified = commentSpecified; } public StructField(String name, Type type) { @@ -65,6 +74,10 @@ public String getComment() { return comment; } + public boolean isCommentSpecified() { + return commentSpecified || !Strings.isNullOrEmpty(comment); + } + public String getName() { return name; } @@ -96,7 +109,7 @@ public String toSql(int depth) { if (type != null) { sb.append(":").append(typeSql); } - if (!Strings.isNullOrEmpty(comment)) { + if (isCommentSpecified()) { sb.append(String.format(" comment '%s'", comment)); } return sb.toString(); @@ -116,7 +129,7 @@ public String prettyPrint(int lpad) { typeStr = typeStr.substring(lpad); sb.append(":").append(typeStr); } - if (!Strings.isNullOrEmpty(comment)) { + if (isCommentSpecified()) { sb.append(String.format(" COMMENT '%s'", comment)); } return sb.toString(); @@ -153,7 +166,7 @@ public String toString() { if (type != null) { sb.append(":").append(type); } - if (!Strings.isNullOrEmpty(comment)) { + if (isCommentSpecified()) { sb.append(String.format(" COMMENT '%s'", comment)); } return sb.toString(); diff --git a/fe/pom.xml b/fe/pom.xml index 2b44718723b05a..fb9b8088f69c7e 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -291,6 +291,7 @@ under the License. 1.2.5 2.0.17 4.0.2 + 2.4.0 4.2.15.Final @@ -918,6 +919,11 @@ under the License. cel ${cel.version} + + io.github.resilience4j + resilience4j-ratelimiter + ${resilience4j.version} + org.javassist diff --git a/fs_brokers/cdc_client/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java b/fs_brokers/cdc_client/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java index bbf973641e4cd0..275ec70921163a 100644 --- a/fs_brokers/cdc_client/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java +++ b/fs_brokers/cdc_client/src/main/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSource.java @@ -84,13 +84,18 @@ import static io.debezium.util.Strings.isNullOrEmpty; /** - * Copied from FlinkCDC project(3.5.0). + * Copied from FlinkCDC project(3.6.0). * - *

Line 924 : change Log Level info to debug. + *

Line 940 : change Log Level info to debug. + * + *

Line 420 : exclude OceanBase heartbeat events from restart event counting. */ public class MySqlStreamingChangeEventSource implements StreamingChangeEventSource { + public static final String EXCLUDE_HEARTBEAT_FROM_EVENT_COUNT = + "doris.cdc.exclude.heartbeat.from.event.count"; + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlStreamingChangeEventSource.class); @@ -103,6 +108,7 @@ public class MySqlStreamingChangeEventSource private final Clock clock; private final EventProcessingFailureHandlingMode eventDeserializationFailureHandlingMode; private final EventProcessingFailureHandlingMode inconsistentSchemaHandlingMode; + private final boolean excludeHeartbeatFromEventCount; private int startingRowNumber = 0; private long initialEventsToSkip = 0L; @@ -226,6 +232,8 @@ public MySqlStreamingChangeEventSource( } } Configuration configuration = connectorConfig.getConfig(); + excludeHeartbeatFromEventCount = + configuration.getBoolean(EXCLUDE_HEARTBEAT_FROM_EVENT_COUNT, false); client.setKeepAlive(configuration.getBoolean(MySqlConnectorConfig.KEEP_ALIVE)); final long keepAliveInterval = configuration.getLong(MySqlConnectorConfig.KEEP_ALIVE_INTERVAL_MS); @@ -408,7 +416,10 @@ protected void handleEvent( eventDispatcher.dispatchHeartbeatEvent(partition, offsetContext); // Capture that we've completed another event ... - offsetContext.completeEvent(); + // OceanBase heartbeat events must not participate in restart event counting. + if (shouldCompleteEvent(excludeHeartbeatFromEventCount, eventType)) { + offsetContext.completeEvent(); + } // update last offset used for logging lastOffset = offsetContext.getOffset(); @@ -521,6 +532,11 @@ protected void handleServerIncident( } } + static boolean shouldCompleteEvent( + boolean excludeHeartbeatFromEventCount, EventType eventType) { + return !excludeHeartbeatFromEventCount || eventType != EventType.HEARTBEAT; + } + /** * Handle the supplied event with a {@link RotateEventData} that signals the logs are being * rotated. This means that either the server was restarted, or the binlog has transitioned to a @@ -937,11 +953,18 @@ private void handleChange( int count = 0; int numRows = rows.size(); if (startingRowNumber < numRows) { - for (int row = startingRowNumber; row != numRows; ++row) { - offsetContext.setRowNumber(row, numRows); - offsetContext.event(tableId, eventTimestamp); - changeEmitter.emit(tableId, rows.get(row)); - count++; + // Use iterator to avoid O(n²) complexity when rows is a LinkedList + // (mysql-binlog-connector-java uses LinkedList in WriteRowsEventDataDeserializer + // and DeleteRowsEventDataDeserializer) + int rowIndex = 0; + for (U rowData : rows) { + if (rowIndex >= startingRowNumber) { + offsetContext.setRowNumber(rowIndex, numRows); + offsetContext.event(tableId, eventTimestamp); + changeEmitter.emit(tableId, rowData); + count++; + } + rowIndex++; } if (LOGGER.isDebugEnabled()) { if (startingRowNumber != 0) { @@ -1273,8 +1296,8 @@ private SSLSocketFactory getBinlogSslSocketFactory( keyManagers = kmf.getKeyManagers(); } catch (KeyStoreException - | NoSuchAlgorithmException - | UnrecoverableKeyException e) { + | NoSuchAlgorithmException + | UnrecoverableKeyException e) { throw new DebeziumException("Could not load keystore", e); } } @@ -1288,23 +1311,23 @@ private SSLSocketFactory getBinlogSslSocketFactory( if (ks == null && (sslMode == SSLMode.PREFERRED || sslMode == SSLMode.REQUIRED)) { trustManagers = new TrustManager[] { - new X509TrustManager() { - - @Override - public void checkClientTrusted( - X509Certificate[] x509Certificates, String s) - throws CertificateException {} - - @Override - public void checkServerTrusted( - X509Certificate[] x509Certificates, String s) - throws CertificateException {} - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } + new X509TrustManager() { + + @Override + public void checkClientTrusted( + X509Certificate[] x509Certificates, String s) + throws CertificateException {} + + @Override + public void checkServerTrusted( + X509Certificate[] x509Certificates, String s) + throws CertificateException {} + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; } + } }; } else { TrustManagerFactory tmf = @@ -1402,7 +1425,6 @@ public GtidSet filterGtidSet( GtidSet mergedGtidSet; if (connectorConfig.gtidNewChannelPosition() == GtidNewChannelPosition.EARLIEST) { - final GtidSet knownGtidSet = filteredGtidSet; LOGGER.info("Using first available positions for new GTID channels"); final GtidSet relevantAvailableServerGtidSet = (gtidSourceFilter != null) @@ -1422,14 +1444,16 @@ public GtidSet filterGtidSet( // recorded offset in the checkpoint, and the available GTID for other MySQL instances // should be completed. mergedGtidSet = - GtidUtils.fixRestoredGtidSet( - GtidUtils.mergeGtidSetInto( - relevantAvailableServerGtidSet.retainAll( - uuid -> knownGtidSet.forServerWithId(uuid) != null), - purgedServerGtid), - filteredGtidSet); + GtidUtils.fixOldChannelsGtidSet( + relevantAvailableServerGtidSet, purgedServerGtid, filteredGtidSet); } else { - mergedGtidSet = availableServerGtidSet.with(filteredGtidSet); + LOGGER.info("Using latest positions for new GTID channels"); + mergedGtidSet = + GtidUtils.computeLatestModeGtidSet( + availableServerGtidSet, + purgedServerGtid, + filteredGtidSet, + gtidSourceFilter); } LOGGER.info("Final merged GTID set to use when connecting to MySQL: {}", mergedGtidSet); diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/DataSource.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/DataSource.java index 904cc32a53702e..fe819b8a5e7ace 100644 --- a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/DataSource.java +++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/DataSource.java @@ -19,5 +19,6 @@ public enum DataSource { MYSQL, - POSTGRES + POSTGRES, + OCEANBASE } diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/SourceReaderFactory.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/SourceReaderFactory.java index 216c2514f42e17..c0eb9b35788b68 100644 --- a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/SourceReaderFactory.java +++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/SourceReaderFactory.java @@ -19,6 +19,7 @@ import org.apache.doris.cdcclient.source.reader.SourceReader; import org.apache.doris.cdcclient.source.reader.mysql.MySqlSourceReader; +import org.apache.doris.cdcclient.source.reader.oceanbase.OceanBaseSourceReader; import org.apache.doris.cdcclient.source.reader.postgres.PostgresSourceReader; import java.util.Map; @@ -38,6 +39,7 @@ public final class SourceReaderFactory { static { register(DataSource.MYSQL, MySqlSourceReader::new); register(DataSource.POSTGRES, PostgresSourceReader::new); + register(DataSource.OCEANBASE, OceanBaseSourceReader::new); } private SourceReaderFactory() {} diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/JdbcIncrementalSourceReader.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/JdbcIncrementalSourceReader.java index a6887ce8a96537..b4c7dcada315b8 100644 --- a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/JdbcIncrementalSourceReader.java +++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/JdbcIncrementalSourceReader.java @@ -733,7 +733,7 @@ protected abstract Fetcher getBinlogSplitReader( splitStart, splitEnd, null, - tableSchemas); + Collections.singletonMap(tableId, tableChange)); return split; } diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java index 430ab7b66681fc..11075ea2d81964 100644 --- a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java +++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java @@ -92,6 +92,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import static io.debezium.connector.mysql.MySqlStreamingChangeEventSource.EXCLUDE_HEARTBEAT_FROM_EVENT_COUNT; import static org.apache.doris.cdcclient.common.Constants.DEBEZIUM_HEARTBEAT_INTERVAL_MS; import static org.apache.doris.cdcclient.utils.ConfigUtil.is13Timestamp; import static org.apache.doris.cdcclient.utils.ConfigUtil.isJson; @@ -102,6 +103,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.shyiko.mysql.binlog.BinaryLogClient; import com.mysql.cj.conf.ConnectionUrl; +import com.mysql.cj.conf.PropertyKey; import io.debezium.connector.mysql.MySqlConnection; import io.debezium.connector.mysql.MySqlConnectorConfig; import io.debezium.connector.mysql.MySqlPartition; @@ -140,6 +142,11 @@ public MySqlSourceReader() { this.snapshotReaderContexts = new CopyOnWriteArrayList<>(); } + /** Whether server heartbeat events should be excluded from the restart event count. */ + protected boolean excludeHeartbeatFromEventCount() { + return false; + } + @Override public void initialize(String jobId, DataSource dataSource, Map config) { this.serializer.init(config); @@ -743,7 +750,7 @@ private MySqlSnapshotSplit createSnapshotSplit( splitStart, splitEnd, null, - tableSchemas); + Collections.singletonMap(tableId, tableChange)); return split; } @@ -994,6 +1001,9 @@ private MySqlSourceConfig generateMySqlConfig( dbzProps.setProperty( MySqlConnectorConfig.KEEP_ALIVE_INTERVAL_MS.name(), DEBEZIUM_HEARTBEAT_INTERVAL_MS + ""); + dbzProps.setProperty( + EXCLUDE_HEARTBEAT_FROM_EVENT_COUNT, + Boolean.toString(excludeHeartbeatFromEventCount())); if (cdcConfig.containsKey(DataSourceConfigKeys.SSL_MODE)) { String normalized = @@ -1018,6 +1028,13 @@ private MySqlSourceConfig generateMySqlConfig( // Keep genuinely ancient (<100) DATE/DATETIME years; MySQL already completes 2-digit years. dbzProps.setProperty("enable.time.adjuster", "false"); + // The converter is valid only when snapshot JDBC exposes YEAR values as numbers. + if ("false" + .equalsIgnoreCase( + jdbcProperteis.getProperty(PropertyKey.yearIsDateType.getKeyName()))) { + dbzProps.setProperty("converters", "dorisYear"); + dbzProps.setProperty("dorisYear.type", MySqlYearConverter.class.getName()); + } configFactory.debeziumProperties(dbzProps); configFactory.heartbeatInterval(Duration.ofMillis(DEBEZIUM_HEARTBEAT_INTERVAL_MS)); diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlYearConverter.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlYearConverter.java new file mode 100644 index 00000000000000..f5aa1646639957 --- /dev/null +++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlYearConverter.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cdcclient.source.reader.mysql; + +import org.apache.kafka.connect.data.SchemaBuilder; + +import java.sql.Date; +import java.util.Properties; + +import io.debezium.spi.converter.CustomConverter; +import io.debezium.spi.converter.RelationalColumn; + +/** Preserves MySQL's special zero YEAR value in both snapshot and binlog records. */ +public class MySqlYearConverter implements CustomConverter { + private static final int BINLOG_ZERO_YEAR = 1900; + + @Override + public void configure(Properties props) {} + + @Override + public void converterFor( + RelationalColumn field, ConverterRegistration registration) { + if (!"YEAR".equalsIgnoreCase(field.typeName())) { + return; + } + String qualifiedColumnName = field.dataCollection() + "." + field.name(); + registration.register( + io.debezium.time.Year.builder(), + value -> MySqlYearConverter.convertYear(value, qualifiedColumnName)); + } + + static Integer convertYear(Object value, String qualifiedColumnName) { + if (value == null) { + return null; + } + if (value instanceof java.time.Year) { + int year = ((java.time.Year) value).getValue(); + return year == BINLOG_ZERO_YEAR ? 0 : year; + } + if (value instanceof Date) { + return ((Date) value).toLocalDate().getYear(); + } + if (value instanceof Number) { + return ((Number) value).intValue(); + } + if (value instanceof String) { + return Integer.valueOf((String) value); + } + throw new IllegalArgumentException( + "Unsupported value type for MySQL YEAR column " + + qualifiedColumnName + + ": " + + value.getClass().getName()); + } +} diff --git a/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/oceanbase/OceanBaseSourceReader.java b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/oceanbase/OceanBaseSourceReader.java new file mode 100644 index 00000000000000..c6509fc14056b4 --- /dev/null +++ b/fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/oceanbase/OceanBaseSourceReader.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cdcclient.source.reader.oceanbase; + +import org.apache.doris.cdcclient.source.reader.mysql.MySqlSourceReader; + +public class OceanBaseSourceReader extends MySqlSourceReader { + + @Override + protected boolean excludeHeartbeatFromEventCount() { + // OceanBase Binlog Service heartbeat events are not guaranteed to replay at the same + // position, so they must not participate in transaction restart event counting. + return true; + } +} diff --git a/fs_brokers/cdc_client/src/test/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSourceTest.java b/fs_brokers/cdc_client/src/test/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSourceTest.java new file mode 100644 index 00000000000000..2f012526b31b65 --- /dev/null +++ b/fs_brokers/cdc_client/src/test/java/io/debezium/connector/mysql/MySqlStreamingChangeEventSourceTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io.debezium.connector.mysql; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.shyiko.mysql.binlog.event.EventType; +import org.junit.jupiter.api.Test; + +class MySqlStreamingChangeEventSourceTest { + + @Test + void oceanBaseHeartbeatDoesNotCompleteEvent() { + assertFalse( + MySqlStreamingChangeEventSource.shouldCompleteEvent( + true, EventType.HEARTBEAT)); + } + + @Test + void oceanBaseBinlogEventCompletesEvent() { + assertTrue( + MySqlStreamingChangeEventSource.shouldCompleteEvent( + true, EventType.WRITE_ROWS)); + } + + @Test + void mysqlHeartbeatKeepsExistingCompletionBehavior() { + assertTrue( + MySqlStreamingChangeEventSource.shouldCompleteEvent( + false, EventType.HEARTBEAT)); + } +} diff --git a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/CdcClientWriteHarness.java b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/CdcClientWriteHarness.java index cdc8dc4a62e63d..07e5b40a811740 100644 --- a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/CdcClientWriteHarness.java +++ b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/CdcClientWriteHarness.java @@ -94,6 +94,57 @@ static CdcClientWriteHarness mysql( String offset, String targetDb, MockDorisServer mock) { + return mysqlCompatible( + jobId, + "MYSQL", + host, + port, + user, + password, + database, + includeTables, + offset, + targetDb, + mock); + } + + static CdcClientWriteHarness oceanbase( + String jobId, + String host, + int port, + String user, + String password, + String database, + String includeTables, + String offset, + String targetDb, + MockDorisServer mock) { + return mysqlCompatible( + jobId, + "OCEANBASE", + host, + port, + user, + password, + database, + includeTables, + offset, + targetDb, + mock); + } + + private static CdcClientWriteHarness mysqlCompatible( + String jobId, + String dataSource, + String host, + int port, + String user, + String password, + String database, + String includeTables, + String offset, + String targetDb, + MockDorisServer mock) { Map config = new HashMap<>(); config.put( DataSourceConfigKeys.JDBC_URL, @@ -108,7 +159,7 @@ static CdcClientWriteHarness mysql( // Point cdc_client's stream-load at the mock BE. Env.getCurrentEnv().setBackendHttpPort(mock.port()); Env.getCurrentEnv().setClusterToken("test"); - return new CdcClientWriteHarness(jobId, "MYSQL", config, targetDb, mock); + return new CdcClientWriteHarness(jobId, dataSource, config, targetDb, mock); } static CdcClientWriteHarness postgres( diff --git a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseSchemaChangeITCase.java b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseSchemaChangeITCase.java new file mode 100644 index 00000000000000..1de7e36cfb0cb4 --- /dev/null +++ b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseSchemaChangeITCase.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cdcclient.itcase; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.doris.cdcclient.common.Env; +import org.apache.doris.job.cdc.split.SnapshotSplit; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.Statement; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +class OceanBaseSchemaChangeITCase extends OceanBaseTestBase { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final AtomicLong JOB_ID_SEQ = new AtomicLong(1_300_000); + + private String jobId; + private String database; + + @BeforeEach + void setUp() throws Exception { + jobId = String.valueOf(JOB_ID_SEQ.incrementAndGet()); + database = "oceanbase_schema_" + jobId; + createDatabase( + database, + "CREATE TABLE t_user (id INT NOT NULL, name VARCHAR(50), PRIMARY KEY (id))", + "INSERT INTO t_user VALUES (1,'alice'), (2,'bob')"); + } + + @AfterEach + void tearDown() throws Exception { + Env.getCurrentEnv().close(jobId); + dropDatabase(database); + } + + @Test + void addAndDropColumnSurviveOffsetCommitFailureAndReaderRebuild() throws Exception { + try (MockDorisServer mock = new MockDorisServer(); + CdcClientWriteHarness harness = + oceanBaseHarness(jobId, database, "t_user", "initial", mock)) { + List splits = harness.fetchAllSnapshotSplits("t_user"); + harness.writeSnapshot(splits); + harness.enterBinlog(splits); + String offsetBeforeAlter = harness.committedOffset(); + String schemasBeforeAlter = harness.committedTableSchemas(); + assertThat(schemasBeforeAlter).isNotNull().doesNotContain("city"); + + execute( + "ALTER TABLE t_user ADD COLUMN city VARCHAR(50)", + "INSERT INTO t_user (id, name, city) VALUES (3, 'carol', 'beijing')"); + + mock.failCommitOffset(); + assertThatThrownBy(() -> harness.continueBinlog(1, Duration.ofSeconds(90))) + .isInstanceOf(Exception.class); + assertThat(mock.failedCommitOffsetCount()).isPositive(); + assertThat(harness.committedOffset()).isEqualTo(offsetBeforeAlter); + assertThat(harness.committedTableSchemas()).isEqualTo(schemasBeforeAlter); + + mock.allowCommitOffset(); + harness.rebuildReaderOnNextWrite(); + List retried = harness.continueBinlog(1, Duration.ofSeconds(90)); + + assertThat(harness.executedDdls()).hasSize(2); + assertThat(harness.executedDdls()).allMatch(ddl -> ddl.contains("ADD COLUMN")); + assertThat(mock.ddlResponses().get(1)) + .contains("Can not add column which already exists"); + assertThat(retried).hasSize(1); + JsonNode added = MAPPER.readTree(retried.get(0)); + assertThat(added.get("id").asInt()).isEqualTo(3); + assertThat(added.get("city").asText()).isEqualTo("beijing"); + assertThat(harness.committedTableSchemas()).contains("city"); + + execute( + "ALTER TABLE t_user DROP COLUMN city", + "INSERT INTO t_user (id, name) VALUES (4, 'dave')"); + List afterDrop = harness.continueBinlog(1, Duration.ofSeconds(90)); + + assertThat(harness.executedDdls().get(2)).contains("DROP COLUMN").contains("city"); + assertThat(afterDrop).hasSize(1); + JsonNode dropped = MAPPER.readTree(afterDrop.get(0)); + assertThat(dropped.get("id").asInt()).isEqualTo(4); + assertThat(dropped.has("city")).isFalse(); + assertThat(harness.committedTableSchemas()).doesNotContain("city"); + } + } + + private void execute(String... statements) throws Exception { + try (Connection connection = connection(database); + Statement statement = connection.createStatement()) { + for (String sql : statements) { + statement.execute(sql); + } + } + } +} diff --git a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseStartupOffsetITCase.java b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseStartupOffsetITCase.java new file mode 100644 index 00000000000000..7b6980cd9adf93 --- /dev/null +++ b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseStartupOffsetITCase.java @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cdcclient.itcase; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.doris.cdcclient.common.Env; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +class OceanBaseStartupOffsetITCase extends OceanBaseTestBase { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final AtomicLong JOB_ID_SEQ = new AtomicLong(1_200_000); + + private String jobId; + private String database; + + @BeforeEach + void setUp() throws Exception { + jobId = String.valueOf(JOB_ID_SEQ.incrementAndGet()); + database = "oceanbase_offset_" + jobId; + createDatabase( + database, + "CREATE TABLE t_user (id INT NOT NULL, name VARCHAR(50), PRIMARY KEY (id))", + "INSERT INTO t_user VALUES (1,'alice'), (2,'bob')"); + } + + @AfterEach + void tearDown() throws Exception { + Env.getCurrentEnv().close(jobId); + dropDatabase(database); + } + + @Test + void latestReadsOnlyChangesAfterReaderStarts() throws Exception { + awaitInitialRowsInBinlog(); + + try (MockDorisServer mock = new MockDorisServer(); + CdcClientWriteHarness harness = + oceanBaseHarness(jobId, database, "t_user", "latest", mock)) { + harness.enterBinlogFromStartupMode(); + insert("INSERT INTO t_user VALUES (3,'carol')"); + + assertThat(ids(harness.continueBinlog(1, Duration.ofSeconds(90)))) + .containsExactly(3); + assertThat(ids(harness.loadedRecords())).doesNotContain(1, 2); + } + } + + @Test + void earliestReplaysExistingBinlogChanges() throws Exception { + try (MockDorisServer mock = new MockDorisServer(); + CdcClientWriteHarness harness = + oceanBaseHarness(jobId, database, "t_user", "earliest", mock)) { + assertThat(ids(harness.readBinlogFromStartupMode(2, Duration.ofSeconds(90)))) + .containsExactlyInAnyOrder(1, 2); + } + } + + @Test + void specificOffsetReplaysChangesAfterRecordedPosition() throws Exception { + awaitInitialRowsInBinlog(); + + String[] position = currentBinlogPosition(); + String offset = + String.format( + "{\"file\":\"%s\",\"pos\":\"%s\"}", position[0], position[1]); + insert("INSERT INTO t_user VALUES (3,'carol')"); + + try (MockDorisServer mock = new MockDorisServer(); + CdcClientWriteHarness harness = + oceanBaseHarness(jobId, database, "t_user", offset, mock)) { + assertThat(ids(harness.readBinlogFromStartupMode(1, Duration.ofSeconds(90)))) + .containsExactly(3); + assertThat(ids(harness.loadedRecords())).doesNotContain(1, 2); + } + } + + private void awaitInitialRowsInBinlog() throws Exception { + String probeJobId = String.valueOf(JOB_ID_SEQ.incrementAndGet()); + + try (MockDorisServer mock = new MockDorisServer(); + CdcClientWriteHarness harness = + oceanBaseHarness( + probeJobId, database, "t_user", "earliest", mock)) { + assertThat( + ids( + harness.readBinlogFromStartupMode( + 2, Duration.ofSeconds(90)))) + .containsExactlyInAnyOrder(1, 2); + } + } + + private String[] currentBinlogPosition() throws Exception { + try (Connection connection = connection(database); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SHOW MASTER STATUS")) { + if (!resultSet.next()) { + throw new IllegalStateException("SHOW MASTER STATUS returned no row"); + } + return new String[] {resultSet.getString("File"), resultSet.getString("Position")}; + } + } + + private void insert(String sql) throws Exception { + try (Connection connection = connection(database); + Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + + private List ids(List records) throws Exception { + List result = new ArrayList<>(); + for (String record : records) { + JsonNode node = MAPPER.readTree(record); + result.add(node.get("id").asInt()); + } + return result; + } +} diff --git a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseTestBase.java b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseTestBase.java new file mode 100644 index 00000000000000..a6157335c3d6a1 --- /dev/null +++ b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseTestBase.java @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cdcclient.itcase; + +import static java.util.concurrent.TimeUnit.SECONDS; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.utility.DockerImageName; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Duration; + +abstract class OceanBaseTestBase { + + protected static final int OCEANBASE_CDC_PORT = 2883; + protected static final String USER = "root@test"; + protected static final String PASSWORD = "123456"; + + private static final String EXTERNAL_HOST = System.getProperty("oceanbase.host"); + private static final int EXTERNAL_CDC_PORT = + Integer.getInteger("oceanbase.cdc.port", OCEANBASE_CDC_PORT); + + static final GenericContainer OCEANBASE = + new GenericContainer<>( + DockerImageName.parse("quay.io/oceanbase/obbinlog-ce:4.2.5-test")) + .withExposedPorts(2881, OCEANBASE_CDC_PORT) + .withStartupTimeout(Duration.ofMinutes(6)) + .waitingFor( + new LogMessageWaitStrategy() + .withRegEx(".*OBBinlog is ready!.*") + .withTimes(1) + .withStartupTimeout(Duration.ofMinutes(6))); + + @BeforeAll + static void initializeOceanBase() { + if (!useExternalOceanBase()) { + OCEANBASE.start(); + } + + Awaitility.await() + .atMost(60, SECONDS) + .pollInterval(1, SECONDS) + .ignoreExceptions() + .until( + () -> { + try (Connection connection = connection(""); + Statement statement = connection.createStatement(); + ResultSet resultSet = + statement.executeQuery("SHOW MASTER STATUS")) { + return connection.isValid(1) + && resultSet.next() + && !resultSet.getString("File").isEmpty() + && resultSet.getLong("Position") >= 4; + } + }); + } + + @AfterAll + static void stopOceanBase() { + if (!useExternalOceanBase()) { + OCEANBASE.stop(); + } + } + + protected static Connection connection(String database) throws Exception { + String url = + "jdbc:mysql://" + + oceanBaseHost() + + ":" + + oceanBaseCdcPort() + + "/" + + database + + "?serverTimezone=UTC"; + return DriverManager.getConnection(url, USER, PASSWORD); + } + + protected void createDatabase(String database, String... statements) throws Exception { + try (Connection connection = connection(""); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE IF EXISTS " + database); + statement.execute("CREATE DATABASE " + database); + statement.execute("USE " + database); + for (String sql : statements) { + statement.execute(sql); + } + } + } + + protected void dropDatabase(String database) throws Exception { + try (Connection connection = connection(""); + Statement statement = connection.createStatement()) { + statement.execute("DROP DATABASE IF EXISTS " + database); + } + } + + protected CdcClientWriteHarness oceanBaseHarness( + String jobId, + String database, + String includeTables, + String offset, + MockDorisServer mock) { + return CdcClientWriteHarness.oceanbase( + jobId, + oceanBaseHost(), + oceanBaseCdcPort(), + USER, + PASSWORD, + database, + includeTables, + offset, + "doris_target_db", + mock); + } + + private static boolean useExternalOceanBase() { + return EXTERNAL_HOST != null; + } + + private static String oceanBaseHost() { + return useExternalOceanBase() ? EXTERNAL_HOST : OCEANBASE.getHost(); + } + + private static int oceanBaseCdcPort() { + return useExternalOceanBase() + ? EXTERNAL_CDC_PORT + : OCEANBASE.getMappedPort(OCEANBASE_CDC_PORT); + } +} diff --git a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseWriteDmlITCase.java b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseWriteDmlITCase.java new file mode 100644 index 00000000000000..eb7159ab7e7178 --- /dev/null +++ b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/OceanBaseWriteDmlITCase.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cdcclient.itcase; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.doris.cdcclient.common.Constants; +import org.apache.doris.cdcclient.common.Env; +import org.apache.doris.job.cdc.split.SnapshotSplit; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.Statement; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +class OceanBaseWriteDmlITCase extends OceanBaseTestBase { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final AtomicLong JOB_ID_SEQ = new AtomicLong(1_100_000); + + private String jobId; + private String database; + + @BeforeEach + void setUp() throws Exception { + jobId = String.valueOf(JOB_ID_SEQ.incrementAndGet()); + database = "oceanbase_dml_" + jobId; + createDatabase( + database, + "CREATE TABLE t_user (id INT NOT NULL, name VARCHAR(50), PRIMARY KEY (id))", + "INSERT INTO t_user VALUES (1,'alice'), (2,'bob'), (3,'carol')"); + } + + @AfterEach + void tearDown() throws Exception { + Env.getCurrentEnv().close(jobId); + dropDatabase(database); + } + + @Test + void snapshotAndBinlogDmlUseOceanBaseReader() throws Exception { + try (MockDorisServer mock = new MockDorisServer(); + CdcClientWriteHarness harness = + oceanBaseHarness(jobId, database, "t_user", "initial", mock)) { + List splits = harness.fetchAllSnapshotSplits("t_user"); + harness.writeSnapshot(splits); + + Map snapshotById = indexById(harness.loadedRecords()); + assertThat(snapshotById).containsOnlyKeys(1, 2, 3); + harness.enterBinlog(splits); + + try (Connection connection = connection(database); + Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO t_user VALUES (4, 'dave')"); + statement.execute("UPDATE t_user SET name = 'alice2' WHERE id = 1"); + statement.execute("DELETE FROM t_user WHERE id = 2"); + } + + List binlog = harness.continueBinlog(3, Duration.ofSeconds(90)); + assertThat(binlog).hasSize(3); + Map binlogById = indexById(binlog); + assertThat(binlogById).containsOnlyKeys(1, 2, 4); + assertThat(binlogById.get(4).get("name").asText()).isEqualTo("dave"); + assertThat(binlogById.get(4).get(Constants.DORIS_DELETE_SIGN).asInt()).isZero(); + assertThat(binlogById.get(1).get("name").asText()).isEqualTo("alice2"); + assertThat(binlogById.get(1).get(Constants.DORIS_DELETE_SIGN).asInt()).isZero(); + assertThat(binlogById.get(2).get(Constants.DORIS_DELETE_SIGN).asInt()).isEqualTo(1); + + harness.rebuildReaderOnNextWrite(); + try (Connection connection = connection(database); + Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO t_user VALUES (5, 'eve')"); + } + + List resumed = harness.continueBinlog(1, Duration.ofSeconds(90)); + assertThat(indexById(resumed)).containsOnlyKeys(5); + } + } + + private Map indexById(List records) throws Exception { + Map result = new HashMap<>(); + for (String record : records) { + JsonNode node = MAPPER.readTree(record); + result.put(node.get("id").asInt(), node); + } + return result; + } +} diff --git a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/factory/SourceReaderFactoryTest.java b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/factory/SourceReaderFactoryTest.java index bee4d048c6222f..8dd8651221ad9a 100644 --- a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/factory/SourceReaderFactoryTest.java +++ b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/factory/SourceReaderFactoryTest.java @@ -22,6 +22,7 @@ import org.apache.doris.cdcclient.source.reader.SourceReader; import org.apache.doris.cdcclient.source.reader.mysql.MySqlSourceReader; +import org.apache.doris.cdcclient.source.reader.oceanbase.OceanBaseSourceReader; import org.apache.doris.cdcclient.source.reader.postgres.PostgresSourceReader; import org.junit.jupiter.api.Test; @@ -42,6 +43,14 @@ void createsPostgresReader() { SourceReaderFactory.createSourceReader(DataSource.POSTGRES)); } + @Test + void createsOceanBaseReader() { + SourceReader reader = SourceReaderFactory.createSourceReader(DataSource.OCEANBASE); + + assertInstanceOf(OceanBaseSourceReader.class, reader); + assertInstanceOf(MySqlSourceReader.class, reader); + } + @Test void eachCallReturnsFreshInstance() { SourceReader first = SourceReaderFactory.createSourceReader(DataSource.MYSQL); diff --git a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReaderTest.java b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReaderTest.java index fe17794f03086f..8905e0379e25a4 100644 --- a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReaderTest.java +++ b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReaderTest.java @@ -20,23 +20,37 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.doris.cdcclient.source.reader.oceanbase.OceanBaseSourceReader; import org.apache.doris.job.cdc.DataSourceConfigKeys; +import org.apache.doris.job.cdc.request.JobBaseConfig; import com.github.shyiko.mysql.binlog.GtidSet; import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig; import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffset; +import org.apache.flink.cdc.connectors.mysql.source.split.MySqlSnapshotSplit; import org.junit.jupiter.api.Test; import java.lang.reflect.Method; +import java.sql.Types; import java.util.HashMap; +import java.util.List; import java.util.Map; +import io.debezium.relational.Column; +import io.debezium.relational.Table; +import io.debezium.relational.TableEditor; +import io.debezium.relational.TableId; +import io.debezium.relational.history.TableChanges; + public class MySqlSourceReaderTest { private static final String SERVER_UUID = "24bc7850-2c16-11e6-a073-0242ac110002"; + private static final String EXCLUDE_HEARTBEAT_FROM_EVENT_COUNT = + "doris.cdc.exclude.heartbeat.from.event.count"; @Test void testNormalizeSslModeMapsAllLegalValues() { @@ -115,6 +129,94 @@ void mysqlConfigCanDisableSchemaChanges() throws Exception { assertFalse(config.isIncludeSchemaChanges()); } + @Test + void mysqlHeartbeatContributesToRestartEventCount() throws Exception { + MySqlSourceConfig config = + sourceConfig(new MySqlSourceReader(), "initial", Map.of()); + + assertFalse( + config.getDbzConfiguration() + .getBoolean(EXCLUDE_HEARTBEAT_FROM_EVENT_COUNT)); + } + + @Test + void oceanBaseHeartbeatDoesNotContributeToRestartEventCount() throws Exception { + MySqlSourceConfig config = + sourceConfig(new OceanBaseSourceReader(), "initial", Map.of()); + + assertTrue( + config.getDbzConfiguration() + .getBoolean(EXCLUDE_HEARTBEAT_FROM_EVENT_COUNT)); + } + + @Test + void snapshotSplitContainsOnlyCurrentTableSchema() throws Exception { + TableId tableId = TableId.parse("testdb.orders"); + TableId otherTableId = TableId.parse("testdb.other_orders"); + TableChanges.TableChange tableChange = tableChange(tableId, "pk"); + Map tableSchemas = new HashMap<>(); + tableSchemas.put(tableId, tableChange); + tableSchemas.put(otherTableId, tableChange(otherTableId, "other_id")); + + MySqlSourceReader reader = + new MySqlSourceReader() { + @Override + protected Class probeSplitKeyClass( + TableId ignoredTableId, + Column ignoredSplitColumn, + JobBaseConfig ignoredJobConfig) { + return Integer.class; + } + }; + reader.setTableSchemas(tableSchemas); + + Method method = + MySqlSourceReader.class.getDeclaredMethod( + "createSnapshotSplit", Map.class, JobBaseConfig.class); + method.setAccessible(true); + MySqlSnapshotSplit split = + (MySqlSnapshotSplit) + method.invoke( + reader, + snapshotOffset("testdb.orders", "pk"), + new JobBaseConfig("job-1", "MYSQL", Map.of(), null)); + + assertEquals(tableId, split.getTableId()); + assertEquals(Map.of(tableId, tableChange), split.getTableSchemas()); + assertFalse(split.getTableSchemas().containsKey(otherTableId)); + assertEquals(2, reader.getTableSchemas().size()); + assertTrue(reader.getTableSchemas().containsKey(tableId)); + assertTrue(reader.getTableSchemas().containsKey(otherTableId)); + } + + @Test + void mysqlConfigRegistersYearConverterOnlyWhenYearIsDateTypeIsFalse() throws Exception { + MySqlSourceConfig falseConfig = + sourceConfig( + "initial", + Map.of( + DataSourceConfigKeys.JDBC_URL, + "jdbc:mysql://localhost:3306/testdb?yearIsDateType=false")); + + assertEquals("dorisYear", falseConfig.getDbzProperties().getProperty("converters")); + assertEquals( + "org.apache.doris.cdcclient.source.reader.mysql.MySqlYearConverter", + falseConfig.getDbzProperties().getProperty("dorisYear.type")); + + MySqlSourceConfig trueConfig = + sourceConfig( + "initial", + Map.of( + DataSourceConfigKeys.JDBC_URL, + "jdbc:mysql://localhost:3306/testdb?yearIsDateType=true")); + assertNull(trueConfig.getDbzProperties().getProperty("converters")); + assertNull(trueConfig.getDbzProperties().getProperty("dorisYear.type")); + + MySqlSourceConfig missingConfig = sourceConfig("initial"); + assertNull(missingConfig.getDbzProperties().getProperty("converters")); + assertNull(missingConfig.getDbzProperties().getProperty("dorisYear.type")); + } + // Drive the real generateMySqlConfig JSON-offset path and return the rebuilt startup offset. private BinlogOffset startupBinlogOffset(String offsetJson) throws Exception { return sourceConfig(offsetJson).getStartupOptions().binlogOffset; @@ -126,6 +228,12 @@ private MySqlSourceConfig sourceConfig(String offset) throws Exception { private MySqlSourceConfig sourceConfig(String offset, Map overrides) throws Exception { + return sourceConfig(new MySqlSourceReader(), offset, overrides); + } + + private MySqlSourceConfig sourceConfig( + MySqlSourceReader reader, String offset, Map overrides) + throws Exception { Map cfg = new HashMap<>(); cfg.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://localhost:3306/testdb"); cfg.put(DataSourceConfigKeys.USER, "u"); @@ -138,6 +246,29 @@ private MySqlSourceConfig sourceConfig(String offset, Map overri MySqlSourceReader.class.getDeclaredMethod( "generateMySqlConfig", Map.class, String.class, int.class); m.setAccessible(true); - return (MySqlSourceConfig) m.invoke(new MySqlSourceReader(), cfg, "job-1", 0); + return (MySqlSourceConfig) m.invoke(reader, cfg, "job-1", 0); + } + + private static Map snapshotOffset(String tableId, String splitKey) { + Map offset = new HashMap<>(); + offset.put("splitId", tableId + ":0"); + offset.put("tableId", tableId); + offset.put("splitKey", List.of(splitKey)); + offset.put("splitStart", new Object[] {1}); + offset.put("splitEnd", new Object[] {10}); + return offset; + } + + private static TableChanges.TableChange tableChange(TableId tableId, String splitKey) { + TableEditor editor = Table.editor().tableId(tableId); + editor.addColumns( + Column.editor() + .name(splitKey) + .type("INT") + .jdbcType(Types.INTEGER) + .optional(false) + .create()); + editor.setPrimaryKeyNames(splitKey); + return new TableChanges.TableChange(TableChanges.TableChangeType.CREATE, editor.create()); } } diff --git a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlYearConverterTest.java b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlYearConverterTest.java new file mode 100644 index 00000000000000..4264567130b579 --- /dev/null +++ b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlYearConverterTest.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cdcclient.source.reader.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import java.sql.Date; +import java.time.LocalDate; +import java.time.Year; + +class MySqlYearConverterTest { + private static final String YEAR_COLUMN = "inventory.orders.order_year"; + + @Test + void preservesSnapshotYearZero() { + assertEquals(0, MySqlYearConverter.convertYear((short) 0, YEAR_COLUMN)); + } + + @Test + void restoresBinlogYearZero() { + assertEquals(0, MySqlYearConverter.convertYear(Year.of(1900), YEAR_COLUMN)); + } + + @Test + void preservesRegularYears() { + assertEquals(2000, MySqlYearConverter.convertYear((short) 2000, YEAR_COLUMN)); + assertEquals(2024, MySqlYearConverter.convertYear(Year.of(2024), YEAR_COLUMN)); + } + + @Test + void acceptsDateFromYearIsDateType() { + assertEquals( + 2024, + MySqlYearConverter.convertYear(Date.valueOf("2024-01-01"), YEAR_COLUMN)); + } + + @Test + void preservesNull() { + assertNull(MySqlYearConverter.convertYear(null, YEAR_COLUMN)); + } + + @Test + void unsupportedTypeIncludesColumnAndType() { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + MySqlYearConverter.convertYear( + LocalDate.of(2024, 1, 1), YEAR_COLUMN)); + + assertEquals( + "Unsupported value type for MySQL YEAR column " + + "inventory.orders.order_year: java.time.LocalDate", + exception.getMessage()); + } +} diff --git a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/postgres/PostgresSourceReaderTest.java b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/postgres/PostgresSourceReaderTest.java index f9f0efe9a73f67..c5119bc8ab2b77 100644 --- a/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/postgres/PostgresSourceReaderTest.java +++ b/fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/reader/postgres/PostgresSourceReaderTest.java @@ -17,22 +17,33 @@ package org.apache.doris.cdcclient.source.reader.postgres; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.doris.cdcclient.source.reader.JdbcIncrementalSourceReader; import org.apache.doris.job.cdc.DataSourceConfigKeys; +import org.apache.doris.job.cdc.request.JobBaseConfig; import io.debezium.config.Configuration; import io.debezium.connector.postgresql.connection.PostgresConnection; import io.debezium.jdbc.JdbcConfiguration; import io.debezium.jdbc.JdbcConnection; +import io.debezium.relational.Column; +import io.debezium.relational.Table; +import io.debezium.relational.TableEditor; +import io.debezium.relational.TableId; +import io.debezium.relational.history.TableChanges; +import org.apache.flink.cdc.connectors.base.source.meta.split.SnapshotSplit; import org.apache.flink.cdc.connectors.postgres.source.config.PostgresSourceConfig; import org.junit.jupiter.api.Test; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.sql.Types; import java.util.HashMap; +import java.util.List; import java.util.Map; class PostgresSourceReaderTest { @@ -72,6 +83,46 @@ void postgresConfigCanDisableSchemaChanges() throws Exception { assertFalse(config.isIncludeSchemaChanges()); } + @Test + void snapshotSplitContainsOnlyCurrentTableSchema() throws Exception { + TableId tableId = TableId.parse("public.orders", false); + TableId otherTableId = TableId.parse("public.other_orders", false); + TableChanges.TableChange tableChange = tableChange(tableId, "pk"); + Map tableSchemas = new HashMap<>(); + tableSchemas.put(tableId, tableChange); + tableSchemas.put(otherTableId, tableChange(otherTableId, "other_id")); + + PostgresSourceReader reader = + new PostgresSourceReader() { + @Override + protected Class probeSplitKeyClass( + TableId ignoredTableId, + Column ignoredSplitColumn, + JobBaseConfig ignoredJobConfig) { + return Integer.class; + } + }; + reader.setTableSchemas(tableSchemas); + + Method method = + JdbcIncrementalSourceReader.class.getDeclaredMethod( + "createSnapshotSplit", Map.class, JobBaseConfig.class); + method.setAccessible(true); + SnapshotSplit split = + (SnapshotSplit) + method.invoke( + reader, + snapshotOffset("public.orders", "pk"), + new JobBaseConfig("job-1", "POSTGRES", Map.of(), null)); + + assertEquals(tableId, split.getTableId()); + assertEquals(Map.of(tableId, tableChange), split.getTableSchemas()); + assertFalse(split.getTableSchemas().containsKey(otherTableId)); + assertEquals(2, reader.getTableSchemas().size()); + assertTrue(reader.getTableSchemas().containsKey(tableId)); + assertTrue(reader.getTableSchemas().containsKey(otherTableId)); + } + private static PostgresSourceConfig sourceConfig(Map overrides) throws Exception { Map cfg = new HashMap<>(); @@ -97,4 +148,27 @@ private static Object connectionFactory(JdbcConnection connection) throws Except field.setAccessible(true); return field.get(connection); } + + private static Map snapshotOffset(String tableId, String splitKey) { + Map offset = new HashMap<>(); + offset.put("splitId", tableId + ":0"); + offset.put("tableId", tableId); + offset.put("splitKey", List.of(splitKey)); + offset.put("splitStart", new Object[] {1}); + offset.put("splitEnd", new Object[] {10}); + return offset; + } + + private static TableChanges.TableChange tableChange(TableId tableId, String splitKey) { + TableEditor editor = Table.editor().tableId(tableId); + editor.addColumns( + Column.editor() + .name(splitKey) + .type("int4") + .jdbcType(Types.INTEGER) + .optional(false) + .create()); + editor.setPrimaryKeyNames(splitKey); + return new TableChanges.TableChange(TableChanges.TableChangeType.CREATE, editor.create()); + } } diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index bff85eee942756..e5c458132a94e1 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -394,6 +394,7 @@ message TxnCoordinatorPB { message RoutineLoadProgressPB { map partition_to_offset = 1; optional RoutineLoadJobStatisticPB stat = 2; + // Error log URLs and first error messages are transient diagnostics and are not stored here. } message RLTaskTxnCommitAttachmentPB { @@ -406,6 +407,7 @@ message RLTaskTxnCommitAttachmentPB { optional int64 task_execution_time_ms = 7; optional RoutineLoadProgressPB progress = 8; optional string error_log_url = 9; + optional string first_error_msg = 10; } message RoutineLoadJobStatisticPB { diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 3a45208ea9bf11..18e152f447df9c 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -487,6 +487,8 @@ struct TIcebergTableSink { 15: optional map static_partition_values; 16: optional PlanNodes.TSortInfo sort_info; 17: optional TIcebergWriteType write_type = TIcebergWriteType.INSERT; + // Unset keeps collection enabled for rolling upgrades with older FEs. + 18: optional bool collect_column_stats; } struct TIcebergRewritableDeleteFileSet { @@ -533,6 +535,8 @@ struct TIcebergMergeSink { 11: optional map hadoop_config 12: optional Types.TFileType file_type 13: optional list broker_addresses; + // Unset keeps collection enabled for rolling upgrades with older FEs. + 14: optional bool collect_column_stats; // delete side (position delete only) 20: optional TFileContent delete_type diff --git a/gensrc/thrift/Exprs.thrift b/gensrc/thrift/Exprs.thrift index a17cd140c93418..812a65740a77c2 100644 --- a/gensrc/thrift/Exprs.thrift +++ b/gensrc/thrift/Exprs.thrift @@ -329,6 +329,10 @@ struct TExprNode { 39: optional bool is_cast_nullable 40: optional TSearchParam search_param 41: optional bool short_circuit_evaluation + // Lambda argument names in the current lambda scope. It is used by BE to + // distinguish current-scope lambda arguments from captured outer lambda + // arguments when nested lambda expressions contain duplicated column ids. + 42: optional list lambda_argument_names } // A flattened representation of a tree of Expr nodes, obtained by depth-first diff --git a/gensrc/thrift/ExternalTableSchema.thrift b/gensrc/thrift/ExternalTableSchema.thrift index 46ae54781ae825..86915e46d28bfb 100644 --- a/gensrc/thrift/ExternalTableSchema.thrift +++ b/gensrc/thrift/ExternalTableSchema.thrift @@ -58,7 +58,10 @@ struct TField { // True when initial_default_value is Base64 and must be decoded before constructing the Doris // STRING/CHAR/VARBINARY value. This cannot be inferred from the Doris type because Iceberg // UUID/BINARY/FIXED may map either to VARBINARY or to STRING/CHAR. - 8: optional bool initial_default_value_is_base64 + 8: optional bool initial_default_value_is_base64, + // Version marker for authoritative Iceberg mapping semantics. Its absence preserves the + // legacy name fallback when a new BE executes a plan produced by an older FE during rollout. + 9: optional bool name_mapping_is_authoritative } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index d87d7df56edd92..33c0a11782155b 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -675,6 +675,7 @@ struct TRLTaskTxnCommitAttachment { 10: optional TKafkaRLTaskProgress kafkaRLTaskProgress 11: optional string errorLogUrl 12: optional TKinesisRLTaskProgress kinesisRLTaskProgress + 13: optional string firstErrorMsg } struct TTxnCommitAttachment { @@ -863,6 +864,7 @@ struct TFrontendPingFrontendResult { 8: optional list diskInfos 9: optional i64 processUUID 10: optional i32 arrowFlightSqlPort + 11: optional string localResourceGroup } struct TPropertyVal { @@ -1668,6 +1670,7 @@ struct TRoutineLoadJob { 19: optional i32 current_abort_task_num 20: optional bool is_abnormal_pause 21: optional string compute_group + 22: optional string first_error_msg } struct TFetchRoutineLoadJobResult { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index b546649933c239..6596c72ba7eeeb 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -560,6 +560,10 @@ struct TFileScanRangeParams { 32: optional map es_docvalue_context // ES fields field→keyword mappings 33: optional map es_fields_context + // Versioned Iceberg scan semantics negotiated by FE. Absence/zero preserves legacy BE + // behavior during a BE-first rolling upgrade; version 1 enables file-wide ID projection and + // logical initial-default materialization. + 34: optional i32 iceberg_scan_semantics_version } struct TFileRangeDesc { @@ -1713,6 +1717,10 @@ struct TPlanNode { 52: optional TRecCTEScanNode rec_cte_scan_node 53: optional TBucketedAggregationNode bucketed_agg_node 54: optional TLocalExchangeNode local_exchange_node + // COUNT(*) and COUNT(col) share push_down_agg_type_opt=COUNT, but file readers need to know + // whether a projected scan slot is the aggregate argument or merely the placeholder retained by + // column pruning. Empty means row-count semantics; non-empty identifies explicit COUNT columns. + 55: optional list push_down_count_slot_ids // projections is final projections, which means projecting into results and materializing them into the output block. 101: optional list projections diff --git a/gensrc/thrift/QueryCache.thrift b/gensrc/thrift/QueryCache.thrift index 048b27f4572dd1..67c31cd4bc6120 100644 --- a/gensrc/thrift/QueryCache.thrift +++ b/gensrc/thrift/QueryCache.thrift @@ -50,4 +50,31 @@ struct TQueryCacheParam { 6: optional i64 entry_max_bytes 7: optional i64 entry_max_rows -} \ No newline at end of file + + // Whether BE is allowed to serve a stale cache entry by incremental merge: + // when the cached version is behind the current version, BE may scan only the + // delta rowsets in (cached_version, current_version], produce the partial + // aggregation of the delta, emit it together with the cached partial blocks + // (the upstream merge aggregation combines both), and write the merged entry + // back with the new version. + // + // FE only sets this to true when all of the following hold, otherwise the + // "cached + delta" union would not equal the new snapshot or could not be + // merged safely: + // - the scanned index is append-only: DUP_KEYS, or merge-on-write + // UNIQUE_KEYS (BE verifies per tablet, through the delete bitmap of the + // delta window, that no pre-existing key was rewritten); merge-on-read + // UNIQUE resolves duplicates while reading and AGG tables merge rows in + // the storage layer, so those always fall back + // - the cache point aggregation does not finalize (its output is a partial + // state that is always merged again by an upstream aggregation, so the + // cached blocks and the delta blocks can be emitted side by side) + // - the cache point aggregates the raw detail rows directly (its child is + // the olap scan node); with a nested aggregation the inner finalized agg + // would see only the delta rows, whose output is not a mergeable + // complement of the cached snapshot + // BE additionally falls back to a full recompute when the delta version path + // cannot be captured (e.g. merged away by compaction), when the delta + // contains delete predicates, or when the delta rewrites history rows. + 8: optional bool allow_incremental +} diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index ec01b743813df3..afbe0b1a5c3ba0 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -135,6 +135,7 @@ doris_port=9030 mariadb_10_port=3326 db2_11_port=50000 oceanbase_port=2881 +oceanbase_cdc_port=2883 // hive catalog test config // To enable hive/paimon test, you need first start hive container. @@ -335,4 +336,4 @@ hudiMinioSecretKey="minio123" icebergDlfRestCatalog="'type' = 'iceberg', 'warehouse' = 'new_dlf_iceberg_catalog', 'iceberg.catalog.type' = 'rest', 'iceberg.rest.uri' = 'http://cn-beijing-vpc.dlf.aliyuncs.com/iceberg', 'iceberg.rest.sigv4-enabled' = 'true', 'iceberg.rest.signing-name' = 'DlfNext', 'iceberg.rest.access-key-id' = 'ak', 'iceberg.rest.secret-access-key' = 'sk', 'iceberg.rest.signing-region' = 'cn-beijing', 'iceberg.rest.vended-credentials-enabled' = 'true', 'io-impl' = 'org.apache.iceberg.rest.DlfFileIO', 'fs.oss.support' = 'true'" // For python UDF test, set the runtime version of python, default: 3.8.10 -// pythonUdfRuntimeVersion = "" \ No newline at end of file +// pythonUdfRuntimeVersion = "" diff --git a/regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load.out b/regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load.out index adc5cddaaad78a..126e5359d6761f 100644 --- a/regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load.out +++ b/regression-test/data/cloud_p0/multi_cluster/stream_load/stream_load.out @@ -11,3 +11,8 @@ -- !all12 -- 11 +-- !http_stream_compute_group -- +40 + +-- !direct_be_compute_group -- +60 diff --git a/regression-test/data/connector_p0/spark_connector/spark_connector_arrow.out b/regression-test/data/connector_p0/spark_connector/spark_connector_arrow.out index 69d0c6629f0ea3..0c7f078f8128a0 100644 --- a/regression-test/data/connector_p0/spark_connector/spark_connector_arrow.out +++ b/regression-test/data/connector_p0/spark_connector/spark_connector_arrow.out @@ -18,13 +18,13 @@ 1 [1, 0, 0, 1, 1] [1, 2, 3] [2, 12, 32] [3, 4, 5, 6] [4, 5, 6] [123456789, 987654321, 123789456] [6.6, 6.7, 7.8] [7.7, 8.800000000000001, 8.899999618530273] [3.12000, 1.12345] ["2023-09-08", "2027-10-28"] ["2023-09-08 17:12:34.123456", "2024-09-08 18:12:34.123456"] ["char", "char2"] ["varchar", "varchar2"] ["string", "string2"] -- !q03 -- -1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} -1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} -1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} -1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} -1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} -1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} -1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} +1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08 17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} +1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08 17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} +1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08 17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} +1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08 17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} +1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08 17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} +1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08 17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} +1 {1:1, 0:1} {1:2, 3:4} {2:4, 5:6} {3:4, 7:8} {4:5, 1:2} {123456789:987654321, 789456123:456789123} {6.6:8.8, 9.9:10.1} {7.7:1.1, 2.2:3.3} {3.12000:1.23000, 2.34000:5.67000} {"2023-09-08":"2024-09-08", "1023-09-08":"2023-09-08"} {"1023-09-08 17:06:51.123456":"2023-09-08 17:12:34.123456", "3023-09-08 17:12:34.123456":"4023-09-08 17:12:34.123456"} {"char":"char2", "char2":"char3"} {"varchar":"varchar2", "varchar3":"varchar4"} {"string":"string2", "string3":"string4"} -- !q04 -- 1 {"c_bool":1, "c_tinyint":1, "c_smallint":2, "c_int":3, "c_bigint":4, "c_largeint":123456789, "c_float":6.6, "c_double":7.7, "c_decimal":3.12000, "c_date":"2023-09-08", "c_datetime":"2023-09-08 17:12:34.123456", "c_char":"char", "c_varchar":"varchar", "c_string":"string"} @@ -34,4 +34,3 @@ 1 {"c_bool":1, "c_tinyint":1, "c_smallint":2, "c_int":3, "c_bigint":4, "c_largeint":123456789, "c_float":6.6, "c_double":7.7, "c_decimal":3.12000, "c_date":"2023-09-08", "c_datetime":"2023-09-08 17:12:34.123456", "c_char":"char", "c_varchar":"varchar", "c_string":"string"} 1 {"c_bool":1, "c_tinyint":1, "c_smallint":2, "c_int":3, "c_bigint":4, "c_largeint":123456789, "c_float":6.6, "c_double":7.7, "c_decimal":3.12000, "c_date":"2023-09-08", "c_datetime":"2023-09-08 17:12:34.123456", "c_char":"char", "c_varchar":"varchar", "c_string":"string"} 1 {"c_bool":1, "c_tinyint":1, "c_smallint":2, "c_int":3, "c_bigint":4, "c_largeint":123456789, "c_float":6.6, "c_double":7.7, "c_decimal":3.12000, "c_date":"2023-09-08", "c_datetime":"2023-09-08 17:12:34.123456", "c_char":"char", "c_varchar":"varchar", "c_string":"string"} - diff --git a/regression-test/data/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.out b/regression-test/data/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.out new file mode 100644 index 00000000000000..55de0e75c37c98 --- /dev/null +++ b/regression-test/data/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !default_value_as_lazy_output -- +\N +\N +\N +90 + +-- !default_value_as_predicate -- +0 +1 +2 +7 + diff --git a/regression-test/data/doc/sql-manual/sql-functions/doc_json_functions_test.out b/regression-test/data/doc/sql-manual/sql-functions/doc_json_functions_test.out index cfa805f95842e7..e55d98e4178454 100644 --- a/regression-test/data/doc/sql-manual/sql-functions/doc_json_functions_test.out +++ b/regression-test/data/doc/sql-manual/sql-functions/doc_json_functions_test.out @@ -45,7 +45,7 @@ ["v1",6.6,[1,2],2] -- !json_extract_no_quotes -- -"doris" +doris -- !json_extract_isnull -- false diff --git a/regression-test/data/empty_relation/eliminate_empty.out b/regression-test/data/empty_relation/eliminate_empty.out index d2f5c2551e1da7..736ff48c267a1d 100644 --- a/regression-test/data/empty_relation/eliminate_empty.out +++ b/regression-test/data/empty_relation/eliminate_empty.out @@ -51,7 +51,7 @@ PhysicalResultSink ----------PhysicalOlapScan[nation] ------PhysicalDistribute[DistributionSpecHash] --------PhysicalProject -----------filter(( not (n_nationkey = 1))) +----------filter(( not (nation.n_nationkey = 1))) ------------PhysicalOlapScan[nation] apply RFs: RF0 -- !except_data_empty_data -- @@ -118,7 +118,7 @@ PhysicalResultSink ----------PhysicalOlapScan[nation] ------PhysicalDistribute[DistributionSpecHash] --------PhysicalProject -----------filter(( not (n_nationkey = 1))) +----------filter(( not (nation.n_nationkey = 1))) ------------PhysicalOlapScan[nation] apply RFs: RF0 -- !null_except_data_empty_data -- diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out index dab31ada31b925..4565871aa8aa71 100644 --- a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out +++ b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out @@ -149,7 +149,7 @@ phone text Yes true \N User phone number grade double Yes true \N email text Yes true \N address struct Yes true \N -col1 double Yes true \N +col1 double Yes true \N Updated column1 type col2 text Yes true \N User defined column2 -- !after_no_comment -- @@ -329,4 +329,3 @@ department_id bigint Yes true \N 2 Bob 30 9223372036854775806 3 Charlie 22 100 3 Charlie 22 9223372036854775805 - diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_mixed_file_formats.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_mixed_file_formats.out new file mode 100644 index 00000000000000..42771b3c4e3891 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_mixed_file_formats.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mixed_file_formats_files -- +orc 3 +parquet 3 + +-- !mixed_file_formats_data -- +1 parquet +2 parquet +3 parquet +4 orc +5 orc +6 orc diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out new file mode 100644 index 00000000000000..93826ea80b7925 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc -- +id int(11) +s struct +arr array> +m map> +arr_scalar array +m_scalar map + +-- !query_rows -- +1 \N 10 \N \N 100 \N \N 1000 \N \N 7 70 +2 first 20 after_a c2 200 202 201 2000 2002 2001 8 80 diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out new file mode 100644 index 00000000000000..54a02e79582045 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !doris_driven_rows -- +1 \N 10 \N doris_before \N \N 100 \N \N \N 1000 \N \N +2 spark_first 20 spark_after_metric spark_after_doris spark_can_write_doris_field 202 200 203 201 2002 2000 2003 2001 + +-- !spark_driven_schema -- +id int(11) +info struct +events array> +attrs map> + +-- !spark_driven_rows_before_write -- +1 \N 10 \N spark_before \N \N 100 \N \N \N 1000 \N \N +2 spark_first_field 20 spark_after_metric_field spark_after spark_new_field 202 200 203 201 2002 2000 2003 2001 + +-- !spark_driven_rows_after_write -- +1 \N 10 \N spark_before \N \N 100 \N \N \N 1000 \N \N +2 spark_first_field 20 spark_after_metric_field spark_after spark_new_field 202 200 203 201 2002 2000 2003 2001 +3 doris_first_field 30 doris_after_metric_field doris_after_spark doris_can_write_spark_field 302 300 303 301 3002 3000 3003 3001 + +-- !required_nested_rows -- +1 10 old-label +2 20 \N + +-- !deep_nested_rows -- +1 last-old middle-old first-old old-note 1.5 12.34 +2 last-new middle-new first-new new-note 2.5 56.78 diff --git a/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out new file mode 100644 index 00000000000000..0ad208e4be8889 --- /dev/null +++ b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !clickhouse_v2_schema -- +2 + +-- !clickhouse_v2_distributed -- +2 + +-- !clickhouse_v2_materialized_view -- +2 diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_empty.csv b/regression-test/data/external_table_p0/tvf/file_scanner_v2_empty.csv new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_0_bigint.parquet b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_0_bigint.parquet new file mode 100644 index 00000000000000..22eff58b7a2389 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_0_bigint.parquet differ diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_1_int.parquet b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_1_int.parquet new file mode 100644 index 00000000000000..5a9ca0baa47e89 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/file_scanner_v2_struct_1_int.parquet differ diff --git a/regression-test/data/external_table_p0/tvf/file_scanner_v2_unsupported_time.parquet b/regression-test/data/external_table_p0/tvf/file_scanner_v2_unsupported_time.parquet new file mode 100644 index 00000000000000..5b973181a56148 Binary files /dev/null and b/regression-test/data/external_table_p0/tvf/file_scanner_v2_unsupported_time.parquet differ diff --git a/regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out b/regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out new file mode 100644 index 00000000000000..c8312571c56e36 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_file_scanner_v2_review_fixes.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !empty_csv -- + +-- !struct_leaf_promotion_cold -- +1 10 100 +3 10 300 + +-- !struct_leaf_promotion_warm -- +1 10 100 +3 10 300 + +-- !count_evolved_struct -- +4 + +-- !unprojected_unsupported_time -- +1 +2 diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out index 4fe42d7fcdcc77..bb74c79d5a6907 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out @@ -808,10 +808,10 @@ true -- !test_98 -- \N \N \N -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 -- !test_100 -- 1317017856 1 18752152 809291 1089176 19951117 3-MEDIUM 0 40 4801000 16034243 9 4368910 72015 3 19951228 RAIL Customer#018752152 q4gN2btSpiKXdN,6 ALGERIA 1 ALGERIA AFRICA 10-753-996-8708 MACHINERY Supplier#001089176 ROidEL1L6yeFsJqnUjD EGYPT 5 EGYPT MIDDLE EAST 14-807-108-7869 blanched gainsboro MFGR#4 MFGR#43 MFGR#433 brown MEDIUM BRUSHED STEEL 42 MED BAG @@ -866,10 +866,10 @@ abcDeFGhijkLmnOp \N 1212 -- !test_107 -- \N \N \N -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -- !test_107_desc -- decimal_flba decimal(5,2) Yes false \N NONE diff --git a/regression-test/data/external_table_p0/tvf/test_tvf_avro.out b/regression-test/data/external_table_p0/tvf/test_tvf_avro.out deleted file mode 100644 index 174d60ce949b88..00000000000000 --- a/regression-test/data/external_table_p0/tvf/test_tvf_avro.out +++ /dev/null @@ -1,97 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !1 -- -aBoolean BOOLEAN Yes false \N NONE -aInt INT Yes false \N NONE -aLong BIGINT Yes false \N NONE -aFloat FLOAT Yes false \N NONE -aDouble DOUBLE Yes false \N NONE -aString TEXT Yes false \N NONE -anArray ARRAY Yes false \N NONE -aMap MAP Yes false \N NONE -anEnum TEXT Yes false \N NONE -aRecord STRUCT Yes false \N NONE -aUnion STRUCT Yes false \N NONE -mapArrayLong MAP> Yes false \N NONE -arrayMapBoolean ARRAY> Yes false \N NONE - --- !2 -- -2 - --- !1 -- -false 100 9999 2.11 9.1102 string test [5, 6, 7] {"k1":1, "k2":2} B {"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} \N {"k11":[77, 11, 33], "k22":[10, 20]} [{"Key11":1}, {"Key22":0}] -true 42 3400 3.14 9.81 a test string [1, 2, 3, 4] {"key1":1, "key2":2} A {"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} \N {"k1":[99, 88, 77], "k2":[10, 20]} [{"arrayMapKey1":0}, {"arrayMapKey2":1}] - --- !2 -- -[1, 2, 3, 4] -[5, 6, 7] - --- !3 -- -{"k1":1, "k2":2} -{"key1":1, "key2":2} - --- !4 -- -A -B - --- !5 -- -{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} -{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} - --- !6 -- -\N -\N - --- !7 -- -{"k1":[99, 88, 77], "k2":[10, 20]} -{"k11":[77, 11, 33], "k22":[10, 20]} - --- !8 -- -[{"Key11":1}, {"Key22":0}] -[{"arrayMapKey1":0}, {"arrayMapKey2":1}] - --- !10 -- -[{"Key11":1}, {"Key22":0}] false {"k1":1, "k2":2} 9999 \N -[{"arrayMapKey1":0}, {"arrayMapKey2":1}] true {"key1":1, "key2":2} 3400 \N - --- !11 -- -a test string 9.81 A -string test 9.1102 B - --- !12 -- -{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} {"k1":1, "k2":2} 2.11 -{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} {"key1":1, "key2":2} 3.14 - --- !3 -- -aBoolean BOOLEAN Yes false \N NONE -aInt INT Yes false \N NONE -aLong BIGINT Yes false \N NONE -aFloat FLOAT Yes false \N NONE -aDouble DOUBLE Yes false \N NONE -aString TEXT Yes false \N NONE -anArray ARRAY Yes false \N NONE -aMap MAP Yes false \N NONE -anEnum TEXT Yes false \N NONE -aRecord STRUCT Yes false \N NONE -aUnion STRUCT Yes false \N NONE -mapArrayLong MAP> Yes false \N NONE -arrayMapBoolean ARRAY> Yes false \N NONE - --- !9 -- -false 100 9999 2.11 9.1102 string test [5, 6, 7] {"k1":1, "k2":2} B {"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} \N {"k11":[77, 11, 33], "k22":[10, 20]} [{"Key11":1}, {"Key22":0}] -true 42 3400 3.14 9.81 a test string [1, 2, 3, 4] {"key1":1, "key2":2} A {"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} \N {"k1":[99, 88, 77], "k2":[10, 20]} [{"arrayMapKey1":0}, {"arrayMapKey2":1}] - --- !4 -- -2 - --- !13 -- -[{"Key11":1}, {"Key22":0}] false {"k1":1, "k2":2} 9999 \N -[{"arrayMapKey1":0}, {"arrayMapKey2":1}] true {"key1":1, "key2":2} 3400 \N - --- !14 -- -a test string 9.81 A -string test 9.1102 B - --- !15 -- -{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} {"k1":1, "k2":2} 2.11 -{"a":5, "b":3.14159265358979, "c":"Simple Record String Field"} {"key1":1, "key2":2} 3.14 - diff --git a/regression-test/data/function_p0/cast/test_cast_null_array_to_nested_array.out b/regression-test/data/function_p0/cast/test_cast_null_array_to_nested_array.out new file mode 100644 index 00000000000000..4a8dd0c8b7263b --- /dev/null +++ b/regression-test/data/function_p0/cast/test_cast_null_array_to_nested_array.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cast_empty_array_to_nested_array_fold -- +[] + +-- !cast_null_array_to_nested_array_fold -- +[null] + +-- !cast_nulls_array_to_nested_array_fold -- +[null, null] + +-- !cast_nested_null_array_to_deeper_array_fold -- +[[null]] + +-- !cast_deeply_nested_null_array_to_deeper_array_fold -- +[[[null]]] + +-- !cast_nested_null_array_to_same_dimension_fold -- +[[null]] + +-- !cast_empty_array_to_nested_array_be -- +[] + +-- !cast_null_array_to_nested_array_be -- +[null] + +-- !cast_nulls_array_to_nested_array_be -- +[null, null] + +-- !cast_nested_null_array_to_deeper_array_be -- +[[null]] + +-- !cast_deeply_nested_null_array_to_deeper_array_be -- +[[[null]]] + +-- !cast_nested_null_array_to_same_dimension_be -- +[[null]] + diff --git a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.out b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.out index e4878ae835914f..0d7e893c805b1a 100644 --- a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.out +++ b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.out @@ -12,7 +12,7 @@ decimal4_u text Yes false \N NONE decimal5_u text Yes false \N NONE double_u double Yes false \N NONE float_u float Yes false \N NONE -boolean boolean Yes false \N NONE +boolean tinyint Yes false \N NONE tinyint tinyint Yes false \N NONE smallint smallint Yes false \N NONE mediumint int Yes false \N NONE @@ -46,9 +46,8 @@ varbinary text Yes false \N NONE enum text Yes false \N NONE -- !select_all_types_null -- -1 120 50000 1000000 9000000000 1000 12345.67 987654.12345 123456789.1234567890 99999999.123456789012345678901234567890 123.45 12.34 true -5 -300 -20000 -500000 -8000000000 -123.45 -12.34 -1000 -1234.56 -98765.43210 -123456789.1234567890 -99999999.123456789012345678901234567890 2023 08:30 08:30:00.123 08:30:00.123456 2023-06-15 2023-06-15T08:30 2023-06-15T08:30 2023-06-15T08:30:00.123 2023-06-15T08:30:00.123456 abc user_001 normal text content c2ltcGxlIGJsb2IgZGF0YQ== {"id": 1, "name": "userA"} Option1 Mw== YmluYXJ5X2RhdGEx dmFyYmluXzAx Value1 +1 120 50000 1000000 9000000000 1000 12345.67 987654.12345 123456789.1234567890 99999999.123456789012345678901234567890 123.45 12.34 1 -5 -300 -20000 -500000 -8000000000 -123.45 -12.34 -1000 -1234.56 -98765.43210 -123456789.1234567890 -99999999.123456789012345678901234567890 0 08:30 08:30:00.123 08:30:00.123456 2023-06-15 2023-06-15T08:30 2023-06-15T08:30 2023-06-15T08:30:00.123 2023-06-15T08:30:00.123456 abc user_001 normal text content c2ltcGxlIGJsb2IgZGF0YQ== {"id": 1, "name": "userA"} Option1 Mw== YmluYXJ5X2RhdGEx dmFyYmluXzAx Value1 -- !select_all_types_null2 -- -1 120 50000 1000000 9000000000 1000 12345.67 987654.12345 123456789.1234567890 99999999.123456789012345678901234567890 123.45 12.34 true -5 -300 -20000 -500000 -8000000000 -123.45 -12.34 -1000 -1234.56 -98765.43210 -123456789.1234567890 -99999999.123456789012345678901234567890 2023 08:30 08:30:00.123 08:30:00.123456 2023-06-15 2023-06-15T08:30 2023-06-15T08:30 2023-06-15T08:30:00.123 2023-06-15T08:30:00.123456 abc user_001 normal text content c2ltcGxlIGJsb2IgZGF0YQ== {"id": 1, "name": "userA"} Option1 Mw== YmluYXJ5X2RhdGEx dmFyYmluXzAx Value1 -2 100 100000 1000000000 100000000000 12345 12345.67 123456789.12345 12345678901234567890.1234567890 123456789012345678901234567890.123456789012345678901234567890 12345.6789 123.456 true -10 -1000 -100000 -100000000 -1000000000000 -12345.6789 -123.456 -12345 -12345.67 -123456789.12345 -12345678901234567890.1234567890 -123456789012345678901234567890.123456789012345678901234567890 2024 12:34:56 12:34:56.789 12:34:56.789123 2024-01-01 2024-01-01T12:34:56 2024-01-01T12:34:56 2024-01-01T12:34:56.789 2024-01-01T12:34:56.789123 hello hello123 this is a text field dGhpcyBpcyBhIGJsb2I= {"id":10,"name":"mock"} Option2 Kg== YmluX2RhdGFfMTIz dmFyYmluX2RhdGE= Value2 - +1 120 50000 1000000 9000000000 1000 12345.67 987654.12345 123456789.1234567890 99999999.123456789012345678901234567890 123.45 12.34 1 -5 -300 -20000 -500000 -8000000000 -123.45 -12.34 -1000 -1234.56 -98765.43210 -123456789.1234567890 -99999999.123456789012345678901234567890 0 08:30 08:30:00.123 08:30:00.123456 2023-06-15 2023-06-15T08:30 2023-06-15T08:30 2023-06-15T08:30:00.123 2023-06-15T08:30:00.123456 abc user_001 normal text content c2ltcGxlIGJsb2IgZGF0YQ== {"id": 1, "name": "userA"} Option1 Mw== YmluYXJ5X2RhdGEx dmFyYmluXzAx Value1 +2 100 100000 1000000000 100000000000 12345 12345.67 123456789.12345 12345678901234567890.1234567890 123456789012345678901234567890.123456789012345678901234567890 12345.6789 123.456 1 -10 -1000 -100000 -100000000 -1000000000000 -12345.6789 -123.456 -12345 -12345.67 -123456789.12345 -12345678901234567890.1234567890 -123456789012345678901234567890.123456789012345678901234567890 2024 12:34:56 12:34:56.789 12:34:56.789123 2024-01-01 2024-01-01T12:34:56 2024-01-01T12:34:56 2024-01-01T12:34:56.789 2024-01-01T12:34:56.789123 hello hello123 this is a text field dGhpcyBpcyBhIGJsb2I= {"id":10,"name":"mock"} Option2 Kg== YmluX2RhdGFfMTIz dmFyYmluX2RhdGE= Value2 diff --git a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_integer_boundary.out b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_integer_boundary.out index c372d368e72906..9b0040492b4c18 100644 --- a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_integer_boundary.out +++ b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_integer_boundary.out @@ -7,25 +7,24 @@ smallint_u int Yes false \N NONE mediumint_u int Yes false \N NONE int_u bigint Yes false \N NONE bigint_u largeint Yes false \N NONE -bool_col boolean Yes false \N NONE +tinyint_1 tinyint Yes false \N NONE tinyint_s tinyint Yes false \N NONE -- !select_snapshot -- -1 all_zero 0 0 0 0 0 false 0 -2 signed_max 127 32767 8388607 2147483647 9223372036854775807 true 127 -3 signed_max_plus1 128 32768 8388608 2147483648 9223372036854775808 false -128 -4 unsigned_max 255 65535 16777215 4294967295 18446744073709551615 true -1 -5 mid_value 100 30000 8000000 2000000000 9000000000000000000 false 50 +1 all_zero 0 0 0 0 0 0 0 +2 signed_max 127 32767 8388607 2147483647 9223372036854775807 1 127 +3 signed_max_plus1 128 32768 8388608 2147483648 9223372036854775808 -1 -128 +4 unsigned_max 255 65535 16777215 4294967295 18446744073709551615 127 -1 +5 mid_value 100 30000 8000000 2000000000 9000000000000000000 4 50 -- !select_after_incr -- -1 all_zero 0 0 0 0 18446744073709551615 false 0 -2 signed_max 127 32767 8388607 2147483647 9223372036854775807 false 127 -3 signed_max_plus1 128 32768 8388608 2147483648 9223372036854775808 false -128 -4 unsigned_max 255 65535 16777215 4294967295 18446744073709551615 true -1 -5 mid_value 100 30000 8000000 4294967295 9000000000000000000 false 50 -101 all_zero 0 0 0 0 0 false 0 -102 signed_max 127 32767 8388607 2147483647 9223372036854775807 true 127 -103 signed_max_plus1 128 32768 8388608 2147483648 9223372036854775808 false -128 -104 unsigned_max 255 65535 16777215 4294967295 18446744073709551615 true -1 -105 mid_value 100 30000 8000000 2000000000 9000000000000000000 false 50 - +1 all_zero 0 0 0 0 18446744073709551615 0 0 +2 signed_max 127 32767 8388607 2147483647 9223372036854775807 4 127 +3 signed_max_plus1 128 32768 8388608 2147483648 9223372036854775808 -1 -128 +4 unsigned_max 255 65535 16777215 4294967295 18446744073709551615 127 -1 +5 mid_value 100 30000 8000000 4294967295 9000000000000000000 4 50 +101 all_zero 0 0 0 0 0 0 0 +102 signed_max 127 32767 8388607 2147483647 9223372036854775807 1 127 +103 signed_max_plus1 128 32768 8388608 2147483648 9223372036854775808 -1 -128 +104 unsigned_max 255 65535 16777215 4294967295 18446744073709551615 127 -1 +105 mid_value 100 30000 8000000 2000000000 9000000000000000000 4 50 diff --git a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_with_concurrent_dml.out b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_with_concurrent_dml.out index 7f1b97bd503a93..e97d6041b53e13 100644 --- a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_with_concurrent_dml.out +++ b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_with_concurrent_dml.out @@ -2,15 +2,27 @@ -- !select_count -- 1007 +-- !select_count_t2 -- +1007 + -- !select_updates -- 1 99 100 99 500 99 999 99 +-- !select_updates_t2 -- +1 99 +100 99 +500 99 +999 99 + -- !select_deletes -- 0 +-- !select_deletes_t2 -- +0 + -- !select_inserts -- 1001 concurrent_ins 1 1002 concurrent_ins 1 @@ -23,3 +35,14 @@ 1009 concurrent_ins 1 1010 concurrent_ins 1 +-- !select_inserts_t2 -- +1001 concurrent_ins 1 +1002 concurrent_ins 1 +1003 concurrent_ins 1 +1004 concurrent_ins 1 +1005 concurrent_ins 1 +1006 concurrent_ins 1 +1007 concurrent_ins 1 +1008 concurrent_ins 1 +1009 concurrent_ins 1 +1010 concurrent_ins 1 diff --git a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job.out b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job.out new file mode 100644 index 00000000000000..26ee9fd67b5102 --- /dev/null +++ b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !oceanbase_snapshot_users -- +1 Alice 18 +2 Bob 20 + +-- !oceanbase_snapshot_orders -- +10 snapshot_order + +-- !oceanbase_snapshot_empty -- + +-- !oceanbase_incremental_users -- +2 Bob 21 +3 Carol 30 + +-- !oceanbase_incremental_orders -- +10 snapshot_order +11 incremental_order diff --git a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_all_type.out b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_all_type.out new file mode 100644 index 00000000000000..2ee538b78663ff --- /dev/null +++ b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_all_type.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !oceanbase_all_type_snapshot -- +1 200 18446744073709551610 123456789.12345 1 2026-07-10 2026-07-10T10:11:12.123456 2026-07-10T10:11:12.123456 abc snapshot snapshot text 41514944 {"id": 1, "name": "snapshot"} A,C NEW 4367734D +2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N + +-- !oceanbase_all_type_incremental -- +1 200 18446744073709551610 1.25000 1 2026-07-10 2026-07-10T10:11:12.123456 2026-07-10T02:11:12.123456 abc updated snapshot text 41514944 {"id":1,"name":"snapshot"} A,C NEW 4367734D +3 100 9000000000 -98765.43210 0 2026-07-11 2026-07-11T11:12:13.654321 2026-07-11T03:12:13.654321 xyz incremental incremental text 2F2B3764 {"id":3,"name":"incremental"} B DONE 71383376 diff --git a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_offset.out b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_offset.out new file mode 100644 index 00000000000000..8786bd3dd061a7 --- /dev/null +++ b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_offset.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !oceanbase_offset_latest -- +2 after_latest + +-- !oceanbase_offset_earliest -- +1 earliest_one +2 earliest_two + +-- !oceanbase_offset_specific -- +10 specific_one +11 specific_two diff --git a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_restart_fe.out b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_restart_fe.out new file mode 100644 index 00000000000000..4f8f344724edc5 --- /dev/null +++ b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_restart_fe.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !oceanbase_restart_fe -- +1 snapshot +2 updated_after_restart +3 after_restart diff --git a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_sc_restart_fe.out b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_sc_restart_fe.out new file mode 100644 index 00000000000000..bbda3141447758 --- /dev/null +++ b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_sc_restart_fe.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !oceanbase_sc_after_restart -- +1 snapshot \N +2 before_restart updated_after_restart +3 after_restart schema_after_restart + +-- !oceanbase_sc_restart_final -- +1 snapshot +2 before_restart +3 after_restart +4 after_drop diff --git a/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_schema_change.out b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_schema_change.out new file mode 100644 index 00000000000000..a4bebceb4c7a8c --- /dev/null +++ b/regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_schema_change.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !oceanbase_after_add -- +1 snapshot \N +2 after_add Hangzhou + +-- !oceanbase_after_drop -- +1 snapshot +2 after_add +3 after_drop diff --git a/regression-test/data/job_p0/streaming_job/cdc/tvf/test_cdc_stream_tvf_mysql.out b/regression-test/data/job_p0/streaming_job/cdc/tvf/test_cdc_stream_tvf_mysql.out index 02b13ffe0b27a7..17277ad47e7022 100644 --- a/regression-test/data/job_p0/streaming_job/cdc/tvf/test_cdc_stream_tvf_mysql.out +++ b/regression-test/data/job_p0/streaming_job/cdc/tvf/test_cdc_stream_tvf_mysql.out @@ -1,11 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_tvf -- -C1 3 -C1 99 -D1 4 -D1 4 +C1 3 4 0 +C1 99 4 0 +D1 4 -1 2024 +D1 4 -1 2024 -- !select_tvf_dml -- -C1 99 -D1 4 - +C1 99 4 0 +D1 4 -1 2024 diff --git a/regression-test/data/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_mysql_jdbc_type_defaults.out b/regression-test/data/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_mysql_jdbc_type_defaults.out new file mode 100644 index 00000000000000..c3c7d34ee6e2ba --- /dev/null +++ b/regression-test/data/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_mysql_jdbc_type_defaults.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !snapshot_data -- +A1 -1 0 +B1 0 2024 +C1 2 0 +D1 4 2025 + +-- !final_data -- +A1 -1 0 +B1 0 2024 +C1 2 0 +D1 4 2025 +E1 127 0 +F1 -128 2026 diff --git a/regression-test/data/nereids_rules_p0/adjust_nullable/test_agg_nullable.out b/regression-test/data/nereids_rules_p0/adjust_nullable/test_agg_nullable.out index 4891c8d8ad0b49..3c0f2d6f394bef 100644 --- a/regression-test/data/nereids_rules_p0/adjust_nullable/test_agg_nullable.out +++ b/regression-test/data/nereids_rules_p0/adjust_nullable/test_agg_nullable.out @@ -4,7 +4,7 @@ -- !agg_nullable_shape -- PhysicalResultSink ---PhysicalProject[AND[k IS NULL,NULL] AS `k > 10 and k < 5`] +--PhysicalProject[AND[s.k IS NULL,NULL] AS `k > 10 and k < 5`] ----hashAgg[GLOBAL] ------PhysicalEmptyRelation diff --git a/regression-test/data/nereids_rules_p0/adjust_nullable/test_subquery_nullable.out b/regression-test/data/nereids_rules_p0/adjust_nullable/test_subquery_nullable.out index cdf32d50f56b4f..38e4352473b9f7 100644 --- a/regression-test/data/nereids_rules_p0/adjust_nullable/test_subquery_nullable.out +++ b/regression-test/data/nereids_rules_p0/adjust_nullable/test_subquery_nullable.out @@ -6,7 +6,7 @@ -- !uncorrelate_scalar_subquery_shape -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----PhysicalProject[AND[x IS NULL,NULL] AS `x > 10 and x < 1`, a AS `a`] +----PhysicalProject[AND[test_subquery_nullable_t2.x IS NULL,NULL] AS `x > 10 and x < 1`, cte1.a AS `a`] ------NestedLoopJoin[CROSS_JOIN] --------PhysicalProject[test_subquery_nullable_t1.a] ----------PhysicalOlapScan[test_subquery_nullable_t1] @@ -25,7 +25,7 @@ PhysicalResultSink -- !correlate_scalar_subquery_shape -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----PhysicalProject[AND[any_value(x) IS NULL,NULL] AS `x > 10 and x < 1`, a AS `a`] +----PhysicalProject[AND[any_value(x) IS NULL,NULL] AS `x > 10 and x < 1`, cte1.a AS `a`] ------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((test_subquery_nullable_t1.b = test_subquery_nullable_t2.y)) otherCondition=() --------PhysicalProject[test_subquery_nullable_t1.a, test_subquery_nullable_t1.b] ----------PhysicalOlapScan[test_subquery_nullable_t1] @@ -43,7 +43,7 @@ PhysicalResultSink -- !uncorrelate_top_nullable_agg_scalar_subquery_shape -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----PhysicalProject[AND[sum(x) IS NULL,NULL] AS `x > 10 and x < 1`, a AS `a`] +----PhysicalProject[AND[sum(x) IS NULL,NULL] AS `x > 10 and x < 1`, cte1.a AS `a`] ------NestedLoopJoin[CROSS_JOIN] --------PhysicalProject[test_subquery_nullable_t1.a] ----------PhysicalOlapScan[test_subquery_nullable_t1] @@ -61,7 +61,7 @@ PhysicalResultSink -- !correlate_top_nullable_agg_scalar_subquery_shape -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----PhysicalProject[AND[sum(x) IS NULL,NULL] AS `x > 10 and x < 1`, a AS `a`] +----PhysicalProject[AND[sum(x) IS NULL,NULL] AS `x > 10 and x < 1`, cte1.a AS `a`] ------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((test_subquery_nullable_t1.b = test_subquery_nullable_t2.y)) otherCondition=() --------PhysicalProject[test_subquery_nullable_t1.a, test_subquery_nullable_t1.b] ----------PhysicalOlapScan[test_subquery_nullable_t1] @@ -79,7 +79,7 @@ PhysicalResultSink -- !uncorrelate_top_notnullable_agg_scalar_subquery_shape -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----PhysicalProject[FALSE AS `x > 10 and x < 1`, a AS `a`] +----PhysicalProject[FALSE AS `x > 10 and x < 1`, cte1.a AS `a`] ------NestedLoopJoin[CROSS_JOIN] --------PhysicalProject[test_subquery_nullable_t1.a] ----------PhysicalOlapScan[test_subquery_nullable_t1] @@ -98,7 +98,7 @@ PhysicalResultSink -- !correlate_top_notnullable_agg_scalar_subquery_shape -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----PhysicalProject[FALSE AS `x > 10 and x < 1`, a AS `a`] +----PhysicalProject[FALSE AS `x > 10 and x < 1`, cte1.a AS `a`] ------PhysicalOlapScan[test_subquery_nullable_t1] -- !uncorrelate_notop_notnullable_agg_scalar_subquery -- @@ -108,7 +108,7 @@ PhysicalResultSink -- !uncorrelate_notop_notnullable_agg_scalar_subquery_shape -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----PhysicalProject[AND[count(x) IS NULL,NULL] AS `x > 10 and x < 1`, a AS `a`] +----PhysicalProject[AND[count(x) IS NULL,NULL] AS `x > 10 and x < 1`, cte1.a AS `a`] ------NestedLoopJoin[CROSS_JOIN] --------PhysicalProject[test_subquery_nullable_t1.a] ----------PhysicalOlapScan[test_subquery_nullable_t1] @@ -127,14 +127,14 @@ PhysicalResultSink -- !correlate_notop_notnullable_agg_scalar_subquery_shape -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----PhysicalProject[AND[any_value(count(x)) IS NULL,NULL] AS `x > 10 and x < 1`, a AS `a`, assert_true(OR[(count(*) <= 1),count(*) IS NULL], 'correlate scalar subquery must return only 1 row') AS `assert_true(OR[count(*) IS NULL,(count(*) <= 1)], 'correlate scalar subquery must return only 1 row')`, assert_true(OR[(count(*) <= 1),count(*) IS NULL], 'correlate scalar subquery must return only 1 row') AS `assert_true(OR[count(*) IS NULL,(count(*) <= 1)], 'correlate scalar subquery must return only 1 row')`] +----PhysicalProject[AND[any_value(count(x)) IS NULL,NULL] AS `x > 10 and x < 1`, assert_true(OR[(count(*) <= 1),count(*) IS NULL], 'correlate scalar subquery must return only 1 row') AS `assert_true(OR[count(*) IS NULL,(count(*) <= 1)], 'correlate scalar subquery must return only 1 row')`, assert_true(OR[(count(*) <= 1),count(*) IS NULL], 'correlate scalar subquery must return only 1 row') AS `assert_true(OR[count(*) IS NULL,(count(*) <= 1)], 'correlate scalar subquery must return only 1 row')`, cte1.a AS `a`] ------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((expr_(cast(a as BIGINT) + cast(b as BIGINT)) = cast(x as BIGINT))) otherCondition=() ---------PhysicalProject[(cast(a as BIGINT) + cast(b as BIGINT)) AS `expr_(cast(a as BIGINT) + cast(b as BIGINT))`, test_subquery_nullable_t1.a] +--------PhysicalProject[(cast(test_subquery_nullable_t1.a as BIGINT) + cast(test_subquery_nullable_t1.b as BIGINT)) AS `expr_(cast(a as BIGINT) + cast(b as BIGINT))`, test_subquery_nullable_t1.a] ----------PhysicalOlapScan[test_subquery_nullable_t1] --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] ---------------PhysicalProject[cast(x as BIGINT) AS `cast(x as BIGINT)`, count(x)] +--------------PhysicalProject[cast(test_subquery_nullable_t2.x as BIGINT) AS `cast(x as BIGINT)`, count(x)] ----------------hashAgg[GLOBAL] ------------------PhysicalProject[test_subquery_nullable_t2.x] --------------------filter((test_subquery_nullable_t2.x > 1000)) diff --git a/regression-test/data/nereids_rules_p0/case_when_rules/test_case_when_rules.out b/regression-test/data/nereids_rules_p0/case_when_rules/test_case_when_rules.out index 686bcfd06f13bd..bc722597f5103a 100644 --- a/regression-test/data/nereids_rules_p0/case_when_rules/test_case_when_rules.out +++ b/regression-test/data/nereids_rules_p0/case_when_rules/test_case_when_rules.out @@ -2,7 +2,7 @@ -- !sql_1_shape -- PhysicalResultSink --PhysicalProject -----filter(a IN (1, 3)) +----filter(tbl_test_case_when_rules.a IN (1, 3)) ------PhysicalOlapScan[tbl_test_case_when_rules] -- !sql_1_result -- @@ -21,7 +21,7 @@ PhysicalResultSink -- !sql_3_shape -- PhysicalResultSink --PhysicalProject -----filter(OR[( not a IN (1, 2, 3)),a IS NULL]) +----filter(OR[( not tbl_test_case_when_rules.a IN (1, 2, 3)),tbl_test_case_when_rules.a IS NULL]) ------PhysicalOlapScan[tbl_test_case_when_rules] -- !sql_3_result -- @@ -33,7 +33,7 @@ PhysicalResultSink -- !sql_4_shape -- PhysicalResultSink --PhysicalProject -----filter(OR[( not (a = 3)),a IS NULL]) +----filter(OR[( not (tbl_test_case_when_rules.a = 3)),tbl_test_case_when_rules.a IS NULL]) ------PhysicalOlapScan[tbl_test_case_when_rules] -- !sql_4_result -- @@ -47,7 +47,7 @@ PhysicalResultSink -- !sql_5_shape -- PhysicalResultSink --PhysicalProject -----filter(a IN (1, 2)) +----filter(tbl_test_case_when_rules.a IN (1, 2)) ------PhysicalOlapScan[tbl_test_case_when_rules] -- !sql_5_result -- diff --git a/regression-test/data/nereids_rules_p0/constant_propagation/constant_propagation.out b/regression-test/data/nereids_rules_p0/constant_propagation/constant_propagation.out index e1f6e5e20f5c7d..9158c626398463 100644 --- a/regression-test/data/nereids_rules_p0/constant_propagation/constant_propagation.out +++ b/regression-test/data/nereids_rules_p0/constant_propagation/constant_propagation.out @@ -17,7 +17,7 @@ PhysicalResultSink -- !project_1_shape -- PhysicalResultSink ---PhysicalProject[40 AS `(d_int + 10) * 2`, d_int AS `a`, t1.d_int] +--PhysicalProject[40 AS `(d_int + 10) * 2`, t1.d_int, t1.d_int AS `a`] ----filter((t1.d_int = 10)) ------PhysicalOlapScan[t1] @@ -54,7 +54,7 @@ PhysicalResultSink --PhysicalProject[t1.a, t1.b, t1.c] ----PhysicalQuickSort[MERGE_SORT, orderKeys=((cast(b as BIGINT) + cast(c as BIGINT)) asc null first)] ------PhysicalQuickSort[LOCAL_SORT, orderKeys=((cast(b as BIGINT) + cast(c as BIGINT)) asc null first)] ---------PhysicalProject[(cast(c as BIGINT) + 2) AS `(cast(b as BIGINT) + cast(c as BIGINT))`, t1.a, t1.b, t1.c] +--------PhysicalProject[(cast(t1.c as BIGINT) + 2) AS `(cast(b as BIGINT) + cast(c as BIGINT))`, t1.a, t1.b, t1.c] ----------filter((t1.a = 1) and (t1.b = 2)) ------------PhysicalOlapScan[t1] @@ -66,7 +66,7 @@ PhysicalResultSink --PhysicalProject[t1.a, t1.b, t1.c] ----PhysicalQuickSort[MERGE_SORT, orderKeys=((cast(a as BIGINT) + cast(b as BIGINT)) asc null first)] ------PhysicalQuickSort[LOCAL_SORT, orderKeys=((cast(a as BIGINT) + cast(b as BIGINT)) asc null first)] ---------PhysicalProject[(cast(a as BIGINT) + 2) AS `(cast(a as BIGINT) + cast(b as BIGINT))`, t1.a, t1.b, t1.c] +--------PhysicalProject[(cast(t1.a as BIGINT) + 2) AS `(cast(a as BIGINT) + cast(b as BIGINT))`, t1.a, t1.b, t1.c] ----------filter((t1.b = 2) and (t1.c = 3)) ------------PhysicalOlapScan[t1] @@ -89,7 +89,7 @@ PhysicalResultSink ------filter((t1.a = 1) and (t1.b = 1)) --------PhysicalOlapScan[t1] ----PhysicalProject[t2.x, t2.y, t2.z] -------filter(((cast(x as BIGINT) + cast(y as BIGINT)) = 1)) +------filter(((cast(t2.x as BIGINT) + cast(t2.y as BIGINT)) = 1)) --------PhysicalOlapScan[t2] -- !join_1_result -- @@ -136,7 +136,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalProject[t1.a, t2.x] ----NestedLoopJoin[LEFT_OUTER_JOIN](a = 10) -------PhysicalProject[(a = 10) AS `(a = 10)`, t1.a] +------PhysicalProject[(t1.a = 10) AS `(a = 10)`, t1.a] --------PhysicalOlapScan[t1] ------PhysicalProject[t2.x] --------filter((t2.x = 1)) @@ -150,9 +150,9 @@ PhysicalResultSink PhysicalResultSink --PhysicalProject[t1.a, t2.x] ----NestedLoopJoin[FULL_OUTER_JOIN](a = 10)(x = 1) -------PhysicalProject[(a = 10) AS `(a = 10)`, t1.a] +------PhysicalProject[(t1.a = 10) AS `(a = 10)`, t1.a] --------PhysicalOlapScan[t1] -------PhysicalProject[(x = 1) AS `(x = 1)`, t2.x] +------PhysicalProject[(t2.x = 1) AS `(x = 1)`, t2.x] --------PhysicalOlapScan[t2] -- !join_6_result -- @@ -177,7 +177,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalProject[t1.a] ----NestedLoopJoin[LEFT_ANTI_JOIN](a = 10) -------PhysicalProject[(a = 10) AS `(a = 10)`, t1.a] +------PhysicalProject[(t1.a = 10) AS `(a = 10)`, t1.a] --------PhysicalOlapScan[t1] ------PhysicalProject[1 AS `1`] --------filter((t2.x = 1)) @@ -194,10 +194,10 @@ PhysicalResultSink --------filter((t1.a = 10)) ----------PhysicalOlapScan[t1] ------PhysicalProject[t2.x] ---------filter(((cast(x as BIGINT) * 10) = 10)) +--------filter(((cast(t2.x as BIGINT) * 10) = 10)) ----------PhysicalOlapScan[t2] apply RFs: RF0 ----PhysicalProject[t3.a] -------filter(((cast(a as BIGINT) * 10) = 10) and (t3.__DORIS_DELETE_SIGN__ = 0)) +------filter(((cast(t3.a as BIGINT) * 10) = 10) and (t3.__DORIS_DELETE_SIGN__ = 0)) --------PhysicalOlapScan[t3] -- !join_9_result -- @@ -397,7 +397,7 @@ PhysicalResultSink ------filter((t1.a = 10)) --------PhysicalOlapScan[t1] ----PhysicalProject[t2.x] -------filter(((cast(x as BIGINT) * 10) = 10)) +------filter(((cast(t2.x as BIGINT) * 10) = 10)) --------PhysicalOlapScan[t2] -- !subquery_7_result -- @@ -412,7 +412,7 @@ PhysicalResultSink ----------PhysicalProject[(cast(a as BIGINT) * 10) AS `k`, t1.b] ------------filter(((cast(a as BIGINT) * 10) = 10)) --------------PhysicalOlapScan[t1] -----------PhysicalProject[cast(x as BIGINT) AS `k`, y AS `b`] +----------PhysicalProject[cast(t2.x as BIGINT) AS `k`, t2.y AS `b`] ------------filter((t2.x = 10)) --------------PhysicalOlapScan[t2] --------PhysicalProject[t3.a] @@ -430,7 +430,7 @@ PhysicalResultSink ------PhysicalProject[(cast(a as BIGINT) * 10) AS `k`, t1.b] --------filter(((cast(a as BIGINT) * 10) = 10)) ----------PhysicalOlapScan[t1] -------PhysicalProject[cast(x as BIGINT) AS `k`, y AS `b`] +------PhysicalProject[cast(t2.x as BIGINT) AS `k`, t2.y AS `b`] --------filter((t2.x = 10)) ----------PhysicalOlapScan[t2] apply RFs: RF0 ----PhysicalProject[t3.a] @@ -447,7 +447,7 @@ PhysicalResultSink ------PhysicalProject[(cast(a as BIGINT) * 10) AS `k`, t1.b] --------filter(((cast(a as BIGINT) * 10) = 10)) ----------PhysicalOlapScan[t1] -------PhysicalProject[cast(x as BIGINT) AS `k`, y AS `b`] +------PhysicalProject[cast(t2.x as BIGINT) AS `k`, t2.y AS `b`] --------filter((t2.x = 10)) ----------PhysicalOlapScan[t2] apply RFs: RF0 ----PhysicalProject[t3.a] @@ -510,7 +510,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalProject[t1.a] ----NestedLoopJoin[LEFT_ANTI_JOIN](a = 10) -------PhysicalProject[(a = 10) AS `(a = 10)`, t1.a] +------PhysicalProject[(t1.a = 10) AS `(a = 10)`, t1.a] --------PhysicalOlapScan[t1] ------PhysicalProject[1 AS `1`] --------filter((t2.x = 1)) @@ -523,7 +523,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalProject[t1.a] ----NestedLoopJoin[LEFT_ANTI_JOIN](a = 10) -------PhysicalProject[(a = 10) AS `(a = 10)`, t1.a] +------PhysicalProject[(t1.a = 10) AS `(a = 10)`, t1.a] --------PhysicalOlapScan[t1] ------PhysicalProject[1 AS `1`] --------filter((t2.x = 20)) @@ -587,8 +587,8 @@ PhysicalResultSink PhysicalResultSink --PhysicalProject[10 AS `a`, count(a)] ----filter((sum(b) > 10)) -------hashAgg[GLOBAL, groupByExpr=(t1.a), outputExpr=(count(10) AS `count(a)`, sum(b) AS `sum(b)`, t1.a)] ---------hashAgg[LOCAL, groupByExpr=(t1.a), outputExpr=(partial_count(10) AS `partial_count(10)`, partial_sum(b) AS `partial_sum(b)`, t1.a)] +------hashAgg[GLOBAL, groupByExpr=(t1.a), outputExpr=(count(10) AS `count(a)`, sum(t1.b) AS `sum(b)`, t1.a)] +--------hashAgg[LOCAL, groupByExpr=(t1.a), outputExpr=(partial_count(10) AS `partial_count(10)`, partial_sum(t1.b) AS `partial_sum(b)`, t1.a)] ----------PhysicalProject[t1.a, t1.b] ------------filter((t1.a = 10)) --------------PhysicalOlapScan[t1] @@ -636,10 +636,10 @@ PhysicalResultSink -- !union_4_shape -- PhysicalResultSink --PhysicalUnion -----PhysicalProject[-1 AS `k2`, 3 AS `k1`, c AS `k3`] +----PhysicalProject[-1 AS `k2`, 3 AS `k1`, t1.c AS `k3`] ------filter((t1.a = 1) and (t1.b = 2) and (t1.c = 3)) --------PhysicalOlapScan[t1] -----PhysicalProject[-1 AS `k2`, 3 AS `k1`, z AS `k3`] +----PhysicalProject[-1 AS `k2`, 3 AS `k1`, t2.z AS `k3`] ------filter((t2.x = 1) and (t2.y = 2) and (t2.z = 3)) --------PhysicalOlapScan[t2] @@ -666,10 +666,10 @@ PhysicalResultSink -- !union_6_shape -- PhysicalResultSink --PhysicalUnion -----PhysicalProject[(cast(a as BIGINT) + cast(b as BIGINT)) AS `k1`, (cast(a as BIGINT) - cast(b as BIGINT)) AS `k2`, c AS `k3`] +----PhysicalProject[(cast(a as BIGINT) + cast(b as BIGINT)) AS `k1`, (cast(a as BIGINT) - cast(b as BIGINT)) AS `k2`, t1.c AS `k3`] ------filter(((cast(a as BIGINT) + cast(b as BIGINT)) = 3) and ((cast(a as BIGINT) - cast(b as BIGINT)) = -1) and (t1.c = 3)) --------PhysicalOlapScan[t1] -----PhysicalProject[(cast(x as BIGINT) + cast(y as BIGINT)) AS `k1`, (cast(x as BIGINT) - cast(y as BIGINT)) AS `k2`, z AS `k3`] +----PhysicalProject[(cast(x as BIGINT) + cast(y as BIGINT)) AS `k1`, (cast(x as BIGINT) - cast(y as BIGINT)) AS `k2`, t2.z AS `k3`] ------filter(((cast(x as BIGINT) + cast(y as BIGINT)) = 3) and ((cast(x as BIGINT) - cast(y as BIGINT)) = -1) and (t2.z = 3)) --------PhysicalOlapScan[t2] @@ -697,7 +697,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalUnion -------PhysicalProject[a AS `a`] +------PhysicalProject[t1.a AS `a`] --------filter((t1.a = 10)) ----------PhysicalOlapScan[t1] @@ -707,7 +707,7 @@ PhysicalResultSink -- !union_10_shape -- PhysicalResultSink --PhysicalUnion -----PhysicalProject[a AS `a`] +----PhysicalProject[t1.a AS `a`] ------filter((t1.a = 10)) --------PhysicalOlapScan[t1] @@ -740,7 +740,7 @@ PhysicalResultSink -- !union_13_shape -- PhysicalResultSink --PhysicalLimit[GLOBAL] -----PhysicalProject[a AS `a`] +----PhysicalProject[t1.a AS `a`] ------filter((t1.a = 1)) --------PhysicalOlapScan[t1] @@ -749,7 +749,7 @@ PhysicalResultSink -- !union_14_shape -- PhysicalResultSink ---PhysicalProject[a AS `a`] +--PhysicalProject[t1.a AS `a`] ----filter((t1.a = 1)) ------PhysicalOlapScan[t1] @@ -760,7 +760,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalUnion -------PhysicalProject[a AS `a`] +------PhysicalProject[t1.a AS `a`] --------filter((t1.a = 1)) ----------PhysicalOlapScan[t1] @@ -770,7 +770,7 @@ PhysicalResultSink -- !union_16_shape -- PhysicalResultSink --PhysicalUnion -----PhysicalProject[a AS `a`] +----PhysicalProject[t1.a AS `a`] ------filter((t1.a = 1)) --------PhysicalOlapScan[t1] @@ -835,7 +835,7 @@ PhysicalResultSink -- !update_1_shape -- PhysicalOlapTableSink ---PhysicalProject[0 AS `__DORIS_DELETE_SIGN__`, 0 AS `__DORIS_VERSION_COL__`, 11 AS `d`, a AS `a`, cast((cast(b as BIGINT) + 1) as INT) AS `b`, cast((cast(b as BIGINT) + 1) as SMALLINT) AS `c`] +--PhysicalProject[0 AS `__DORIS_DELETE_SIGN__`, 0 AS `__DORIS_VERSION_COL__`, 11 AS `d`, cast((cast(b as BIGINT) + 1) as INT) AS `b`, cast((cast(b as BIGINT) + 1) as SMALLINT) AS `c`, t3.a AS `a`] ----filter((t3.__DORIS_DELETE_SIGN__ = 0) and (t3.a = 1)) ------PhysicalOlapScan[t3] @@ -846,7 +846,7 @@ PhysicalOlapTableSink -- !update_2_shape -- PhysicalOlapTableSink ---PhysicalProject[0 AS `__DORIS_DELETE_SIGN__`, 0 AS `__DORIS_VERSION_COL__`, a AS `a`, a AS `b`, cast(a as BIGINT) AS `d`, cast(a as SMALLINT) AS `c`] +--PhysicalProject[0 AS `__DORIS_DELETE_SIGN__`, 0 AS `__DORIS_VERSION_COL__`, cast(t3.a as BIGINT) AS `d`, cast(t3.a as SMALLINT) AS `c`, t3.a AS `a`, t3.a AS `b`] ----filter((t3.__DORIS_DELETE_SIGN__ = 0) and (t3.a = 1)) ------PhysicalOlapScan[t3] @@ -857,8 +857,8 @@ PhysicalOlapTableSink -- !no_replace_1_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(count(*) AS `count(1)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_count(*) AS `partial_count(*)`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(count() AS `count(1)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_count() AS `partial_count(*)`)] ------PhysicalProject[1 AS `1`] --------filter((t1.d_string = '012345') and (t1.d_string MATCH_ANY '012345')) ----------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out b/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out index 60fa2fe2c30d9f..848292fa2c0c39 100644 --- a/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out +++ b/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out @@ -6,17 +6,17 @@ PhysicalResultSink -- !eliminate_not_null_in_clause -- PhysicalResultSink ---filter(id IN (1, 2, 3)) +--filter(t.id IN (1, 2, 3)) ----PhysicalOlapScan[t] -- !eliminate_not_null_not_equal -- PhysicalResultSink ---filter(( not (score = 13))) +--filter(( not (t.score = 13))) ----PhysicalOlapScan[t] -- !eliminate_not_null_string_function -- PhysicalResultSink ---filter((length(name) > 0)) +--filter((length(t.name) > 0)) ----PhysicalOlapScan[t] -- !eliminate_not_null_aggregate -- @@ -32,23 +32,23 @@ PhysicalResultSink -- !eliminate_not_null_math_function -- PhysicalResultSink ---filter((abs(score) = 5)) +--filter((abs(t.score) = 5)) ----PhysicalOlapScan[t] -- !eliminate_not_null_complex_logic -- PhysicalResultSink ---filter(( not score IS NULL) and OR[(t.score > 5),(t.id < 10)]) +--filter(( not t.score IS NULL) and OR[(t.score > 5),(t.id < 10)]) ----PhysicalOlapScan[t] -- !eliminate_not_null_date_function -- PhysicalResultSink ---filter((year(cast(name as DATEV2)) = 2022)) +--filter((year(cast(t.name as DATEV2)) = 2022)) ----PhysicalOlapScan[t] -- !eliminate_not_null_with_subquery -- PhysicalResultSink --hashJoin[LEFT_SEMI_JOIN] hashCondition=((t.score = t.score)) otherCondition=() -----filter(( not score IS NULL) and (t.score > 0)) +----filter(( not t.score IS NULL) and (t.score > 0)) ------PhysicalOlapScan[t] ----filter((t.score > 0)) ------PhysicalOlapScan[t] diff --git a/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out b/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out index 1031281c82596b..8724d788b48936 100644 --- a/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out +++ b/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out @@ -113,7 +113,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() -------filter(( not id IS NULL) and (t1.score > 5)) +------filter(( not t1.id IS NULL) and (t1.score > 5)) --------PhysicalOlapScan[t(t1)] ------PhysicalOlapScan[t(t2)] @@ -122,7 +122,7 @@ PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[RIGHT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ------PhysicalOlapScan[t(t1)] -------filter(( not id IS NULL) and (t2.score > 5)) +------filter(( not t2.id IS NULL) and (t2.score > 5)) --------PhysicalOlapScan[t(t2)] -- !full_outer_join_compound_conditions -- @@ -141,7 +141,7 @@ PhysicalResultSink --------filter((t1.score > 5)) ----------PhysicalOlapScan[t(t1)] --------PhysicalOlapScan[t(t2)] -------filter(( not score IS NULL)) +------filter(( not t3.score IS NULL)) --------PhysicalOlapScan[t(t3)] -- !using_non_equijoin_conditions -- @@ -150,7 +150,7 @@ PhysicalResultSink ----PhysicalProject ------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((expr_cast(score as BIGINT) = expr_(cast(score as BIGINT) + 10))) otherCondition=() --------PhysicalProject -----------filter(( not id IS NULL)) +----------filter(( not t1.id IS NULL)) ------------PhysicalOlapScan[t(t1)] --------PhysicalProject ----------PhysicalOlapScan[t(t2)] @@ -172,7 +172,7 @@ PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----PhysicalProject ------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ---------filter(( not id IS NULL)) +--------filter(( not sub.id IS NULL)) ----------PhysicalOlapScan[t(t1)] --------PhysicalProject ----------PhysicalOlapScan[t(t2)] @@ -183,7 +183,7 @@ PhysicalResultSink ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ------filter((t1.score > 10)) --------PhysicalOlapScan[t(t1)] -------filter(( not name IS NULL)) +------filter(( not t2.name IS NULL)) --------PhysicalOlapScan[t(t2)] -- !right_outer -- @@ -192,7 +192,7 @@ PhysicalResultSink ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ------filter((t1.score > 10)) --------PhysicalOlapScan[t(t1)] -------filter(( not name IS NULL)) +------filter(( not t2.name IS NULL)) --------PhysicalOlapScan[t(t2)] -- !full_outer -- @@ -201,7 +201,7 @@ PhysicalResultSink ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ------filter((t1.score > 10)) --------PhysicalOlapScan[t(t1)] -------filter(( not name IS NULL)) +------filter(( not t2.name IS NULL)) --------PhysicalOlapScan[t(t2)] -- !self_left_outer -- @@ -226,7 +226,7 @@ PhysicalResultSink -- !full_outer_multiple_tables -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----filter(name IS NULL) +----filter(t3.name IS NULL) ------hashJoin[FULL_OUTER_JOIN shuffle] hashCondition=((t1.id = t3.id)) otherCondition=() --------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ----------PhysicalOlapScan[t(t1)] @@ -270,6 +270,6 @@ PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ------PhysicalOlapScan[t(t1)] -------filter(( not name IS NULL)) +------filter(( not t2.name IS NULL)) --------PhysicalOlapScan[t(t2)] diff --git a/regression-test/data/nereids_rules_p0/expression/test_disable_nereids_expression_rule.out b/regression-test/data/nereids_rules_p0/expression/test_disable_nereids_expression_rule.out index a9cf8afc2e5cc1..f3c1628136220c 100644 --- a/regression-test/data/nereids_rules_p0/expression/test_disable_nereids_expression_rule.out +++ b/regression-test/data/nereids_rules_p0/expression/test_disable_nereids_expression_rule.out @@ -5,6 +5,6 @@ PhysicalResultSink -- !shape_2 -- PhysicalResultSink ---filter((cast(a as DECIMALV3(11, 1)) = 1.1)) +--filter((cast(test_disable_nereids_expression_rule_tbl.a as DECIMALV3(11, 1)) = 1.1)) ----PhysicalOlapScan[test_disable_nereids_expression_rule_tbl] diff --git a/regression-test/data/nereids_rules_p0/expression/test_simplify_comparison_predicate_int_vs_double.out b/regression-test/data/nereids_rules_p0/expression/test_simplify_comparison_predicate_int_vs_double.out index 4bff663fa577f4..7791adba661df7 100644 --- a/regression-test/data/nereids_rules_p0/expression/test_simplify_comparison_predicate_int_vs_double.out +++ b/regression-test/data/nereids_rules_p0/expression/test_simplify_comparison_predicate_int_vs_double.out @@ -4,7 +4,7 @@ PhysicalResultSink --NestedLoopJoin[LEFT_OUTER_JOIN] ----filter((t1.c_s = '870479087484055553')) ------PhysicalOlapScan[tbl1_test_simplify_comparison_predicate_int_vs_double(t1)] -----filter((cast(c_bigint as DOUBLE) = 8.7047908748405555E17)) +----filter((cast(t2.c_bigint as DOUBLE) = 8.7047908748405555E17)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double(t2)] -- !cast_bigint_as_double_result -- @@ -13,13 +13,13 @@ PhysicalResultSink -- !float_neg16777216_neg10 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = -1.6777226E7)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = -1.6777226E7)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !float_neg16777216_0 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = -1.6777216E7)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = -1.6777216E7)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !float_neg16777216_10 -- @@ -37,37 +37,37 @@ PhysicalResultSink -- !float_16777216_0 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = 1.6777216E7)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = 1.6777216E7)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !float_16777216_10 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = 1.6777226E7)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = 1.6777226E7)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !double_neg9007199254740992_neg10 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as DOUBLE) = -9.007199254741002E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as DOUBLE) = -9.007199254741002E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !float_neg9007199254740992_neg10 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = -9.0071993E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = -9.0071993E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !double_neg9007199254740992_0 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as DOUBLE) = -9.007199254740992E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as DOUBLE) = -9.007199254740992E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !float_neg9007199254740992_0 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = -9.0071993E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = -9.0071993E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !double_neg9007199254740992_10 -- @@ -79,7 +79,7 @@ PhysicalResultSink -- !float_neg9007199254740992_10 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = -9.0071993E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = -9.0071993E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !double_9007199254740992_neg10 -- @@ -91,31 +91,31 @@ PhysicalResultSink -- !float_9007199254740992_neg10 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = 9.0071993E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = 9.0071993E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !double_9007199254740992_0 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as DOUBLE) = 9.007199254740992E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as DOUBLE) = 9.007199254740992E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !float_9007199254740992_0 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = 9.0071993E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = 9.0071993E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !double_9007199254740992_10 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as DOUBLE) = 9.007199254741002E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as DOUBLE) = 9.007199254741002E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !float_9007199254740992_10 -- PhysicalResultSink --PhysicalProject -----filter((cast(c_bigint as FLOAT) = 9.0071993E15)) +----filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as FLOAT) = 9.0071993E15)) ------PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !int_vs_double_1_shape -- @@ -128,7 +128,7 @@ PhysicalResultSink -- !int_vs_double_2_shape -- PhysicalResultSink ---filter((cast(c_bigint as DECIMALV3(7, 3)) > 123.456)) +--filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_bigint as DECIMALV3(7, 3)) > 123.456)) ----PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !int_vs_double_2_result -- @@ -143,14 +143,14 @@ PhysicalResultSink -- !decimal_vs_decimal_2_shape -- PhysicalResultSink ---filter((cast(c_decimal as DECIMALV3(3, 1)) > 12.3)) +--filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_decimal as DECIMALV3(3, 1)) > 12.3)) ----PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !decimal_vs_decimal_2_result -- -- !decimal_vs_decimal_3_shape -- PhysicalResultSink ---filter((cast(c_decimal as DECIMALV3(5, 2)) > 123.45)) +--filter((cast(tbl2_test_simplify_comparison_predicate_int_vs_double.c_decimal as DECIMALV3(5, 2)) > 123.45)) ----PhysicalOlapScan[tbl2_test_simplify_comparison_predicate_int_vs_double] -- !decimal_vs_decimal_3_result -- diff --git a/regression-test/data/nereids_rules_p0/expression/test_simplify_range.out b/regression-test/data/nereids_rules_p0/expression/test_simplify_range.out index cda4a2fb326469..3175e7e9034c5c 100644 --- a/regression-test/data/nereids_rules_p0/expression/test_simplify_range.out +++ b/regression-test/data/nereids_rules_p0/expression/test_simplify_range.out @@ -23,7 +23,7 @@ PhysicalResultSink ----hashAgg[GLOBAL] ------hashAgg[LOCAL] --------PhysicalProject -----------filter(( not (cast(b as BIGINT) * 0) IS NULL)) +----------filter(( not (cast(test_simplify_range_tbl_1.b as BIGINT) * 0) IS NULL)) ------------PhysicalOlapScan[test_simplify_range_tbl_1] -- !sql_3_result -- diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out index 51c751b763006c..88d33929e7e534 100644 --- a/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out +++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_down_filter_other_condition.out @@ -190,7 +190,7 @@ PhysicalResultSink -- !pushdown_left_outer_join_subquery -- PhysicalResultSink ---filter(OR[(cast(id as BIGINT) = sum(id)),id IS NULL]) +--filter(OR[(cast(t1.id as BIGINT) = sum(id)),t1.id IS NULL]) ----NestedLoopJoin[LEFT_OUTER_JOIN](id = 1) ------PhysicalOlapScan[t1] ------hashAgg[GLOBAL] diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out index 355b4b231fc7af..03ea2689a6d8b4 100644 --- a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out +++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out @@ -65,39 +65,3 @@ PhysicalResultSink ----PhysicalOlapScan[t1] ----PhysicalOlapScan[t2] --- !pushdown_cross_join_combine -- -PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) -----PhysicalOlapScan[t1] -----PhysicalOlapScan[t2] - --- !pushdown_inner_join_combine -- -PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) -----PhysicalOlapScan[t1] -----PhysicalOlapScan[t2] - --- !pushdown_left_join_combine -- -PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) -----PhysicalOlapScan[t1] -----PhysicalOlapScan[t2] - --- !pushdown_right_join_combine -- -PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t2.msg = t1.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) -----PhysicalOlapScan[t1] -----PhysicalOlapScan[t2] - --- !pushdown_full_join_combine -- -PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) -----PhysicalOlapScan[t1] -----PhysicalOlapScan[t2] - --- !pushdown_cross_join_combine -- -PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) -----PhysicalOlapScan[t1] -----PhysicalOlapScan[t2] - diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out index f1acddd25757b5..d04e95189da51e 100644 --- a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out +++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_through.out @@ -49,7 +49,7 @@ PhysicalResultSink -- !filter_join_inner -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() -----filter((length(msg) = 0)) +----filter((length(t1.msg) = 0)) ------PhysicalOlapScan[t1] ----PhysicalOlapScan[t2] @@ -228,7 +228,7 @@ PhysicalResultSink -- !filter_aggregation_group_set -- PhysicalResultSink ---filter((cast(msg as DECIMALV3(38, 6)) = 1.000000)) +--filter((cast(t1.msg as DECIMALV3(38, 6)) = 1.000000)) ----hashAgg[GLOBAL] ------PhysicalRepeat --------filter((t1.id > 10)) @@ -236,7 +236,7 @@ PhysicalResultSink -- !filter_aggregation_group_set -- PhysicalResultSink ---filter(OR[(t1.id > 10),(cast(msg as DECIMALV3(38, 6)) = 1.000000)]) +--filter(OR[(t1.id > 10),(cast(t1.msg as DECIMALV3(38, 6)) = 1.000000)]) ----hashAgg[GLOBAL] ------PhysicalRepeat --------PhysicalOlapScan[t1] @@ -304,7 +304,7 @@ PhysicalResultSink --PhysicalUnion ----filter(id IN (2, 3)) ------PhysicalOneRowRelation -----filter(id IN (2, 3)) +----filter(t2.id IN (2, 3)) ------PhysicalOlapScan[t2] -- !push_filter_intersect -- @@ -312,16 +312,16 @@ PhysicalResultSink --PhysicalIntersect ----filter(id IN (2, 3)) ------PhysicalOneRowRelation -----filter(id IN (2, 3)) +----filter(t2.id IN (2, 3)) ------PhysicalOlapScan[t2] -- !push_filter_except -- PhysicalResultSink --PhysicalExcept ----filter((id = 2)) -------filter((length(msg) = 0)) +------filter((length(t1.msg) = 0)) --------PhysicalOlapScan[t1] -----filter((length(msg) = 0) and (t2.id = 2)) +----filter((length(t2.msg) = 0) and (t2.id = 2)) ------PhysicalOlapScan[t2] -- !push_filter_except -- @@ -366,6 +366,6 @@ PhysicalResultSink ----PhysicalQuickSort[LOCAL_SORT] ------PhysicalWindow --------PhysicalQuickSort[LOCAL_SORT] -----------filter(OR[(length(msg) = 0),(t1.id = 2)]) +----------filter(OR[(length(t1.msg) = 0),(t1.id = 2)]) ------------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/extend_infer_equal_predicate.out b/regression-test/data/nereids_rules_p0/infer_predicate/extend_infer_equal_predicate.out index 450cda423d8d62..ae4a51661d6c14 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/extend_infer_equal_predicate.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/extend_infer_equal_predicate.out @@ -17,9 +17,9 @@ PhysicalResultSink -- !test_simple_compare_not_equal -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_int = t2.d_int)) otherCondition=() -----filter(( not (d_int = 10))) +----filter(( not (t1.d_int = 10))) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter(( not (d_int = 10))) +----filter(( not (t2.d_int = 10))) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_simple_compare_datetimev2 -- @@ -33,48 +33,48 @@ PhysicalResultSink -- !test_simple_compare_not_equal_datetimev2 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_datetimev2 = t2.d_datetimev2)) otherCondition=() -----filter(( not (d_datetimev2 = '2024-01-01 00:00:00'))) +----filter(( not (t1.d_datetimev2 = '2024-01-01 00:00:00'))) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter(( not (d_datetimev2 = '2024-01-01 00:00:00'))) +----filter(( not (t2.d_datetimev2 = '2024-01-01 00:00:00'))) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_not_in -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_int = t2.d_int)) otherCondition=() -----filter(( not d_int IN (10, 20))) +----filter(( not t1.d_int IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter(( not d_int IN (10, 20))) +----filter(( not t2.d_int IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_in -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_int = t2.d_int)) otherCondition=() -----filter(d_int IN (10, 20)) +----filter(t1.d_int IN (10, 20)) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter(d_int IN (10, 20)) +----filter(t2.d_int IN (10, 20)) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_func_not_in -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_int = t2.d_int)) otherCondition=() -----filter(( not abs(d_int) IN (10, 20))) +----filter(( not abs(t1.d_int) IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter(( not abs(d_int) IN (10, 20))) +----filter(( not abs(t2.d_int) IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_like -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_char100 = t2.d_char100)) otherCondition=() -----filter((d_char100 like '012%')) +----filter((t1.d_char100 like '012%')) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter((d_char100 like '012%')) +----filter((t2.d_char100 like '012%')) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_like_not -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_char100 = t2.d_char100)) otherCondition=() ----PhysicalOlapScan[extend_infer_t1(t1)] -----filter(( not (d_char100 like '012%'))) +----filter(( not (t2.d_char100 like '012%'))) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_like_to_equal -- @@ -88,9 +88,9 @@ PhysicalResultSink -- !test_func_not_in_and_func_equal_condition -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_abs(d_int) = expr_abs(d_int))) otherCondition=() -----filter(( not abs(d_int) IN (10, 20))) +----filter(( not abs(t1.d_int) IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter(( not abs(d_int) IN (10, 20))) +----filter(( not abs(t2.d_int) IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_between_and -- @@ -128,17 +128,17 @@ PhysicalResultSink -- !test_sign_predicate -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.a = t2.a)) otherCondition=() -----filter((sign(cast(a as DOUBLE)) >= 1)) +----filter((sign(cast(t1.a as DOUBLE)) >= 1)) ------PhysicalOlapScan[extend_infer_t3(t1)] -----filter((sign(cast(a as DOUBLE)) >= 1)) +----filter((sign(cast(t2.a as DOUBLE)) >= 1)) ------PhysicalOlapScan[extend_infer_t4(t2)] -- !test_if_predicate -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_int = t2.d_int)) otherCondition=() -----filter(( not d_int IN (10, 20))) +----filter(( not t1.d_int IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter(( not d_int IN (10, 20))) +----filter(( not t2.d_int IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_if_and_in_predicate -- @@ -160,15 +160,15 @@ PhysicalResultSink -- !test_multi_slot_in_predicate1 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_(cast(a as BIGINT) + cast(c as BIGINT)) = expr_(cast(a as BIGINT) + cast(c as BIGINT)))) otherCondition=() -----filter(((cast(a as BIGINT) + cast(c as BIGINT)) < 10)) +----filter(((cast(t1.a as BIGINT) + cast(t1.c as BIGINT)) < 10)) ------PhysicalOlapScan[extend_infer_t3(t1)] -----filter(((cast(a as BIGINT) + cast(c as BIGINT)) < 10)) +----filter(((cast(t2.a as BIGINT) + cast(t2.c as BIGINT)) < 10)) ------PhysicalOlapScan[extend_infer_t4(t2)] -- !test_multi_slot_in_predicate2 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.a = t2.a) and (t1.b = t2.b)) otherCondition=() -----filter(((cast(a as DOUBLE) + cast(b as DOUBLE)) < 10.0)) +----filter(((cast(t1.a as DOUBLE) + cast(t1.b as DOUBLE)) < 10.0)) ------PhysicalOlapScan[extend_infer_t3(t1)] ----PhysicalOlapScan[extend_infer_t4(t2)] @@ -183,45 +183,45 @@ PhysicalResultSink -- !test_datetimev2_predicate -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_datetimev2 = t2.d_datetimev2)) otherCondition=() -----filter((convert_tz(date_trunc(d_datetimev2, 'month'), 'Asia/Shanghai', 'Europe/Paris') = '2024-01-01 00:00:00')) +----filter((convert_tz(date_trunc(t1.d_datetimev2, 'month'), 'Asia/Shanghai', 'Europe/Paris') = '2024-01-01 00:00:00')) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter((convert_tz(date_trunc(d_datetimev2, 'month'), 'Asia/Shanghai', 'Europe/Paris') = '2024-01-01 00:00:00')) +----filter((convert_tz(date_trunc(t2.d_datetimev2, 'month'), 'Asia/Shanghai', 'Europe/Paris') = '2024-01-01 00:00:00')) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_convert_tz_predicate -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_datetimev2 = t2.d_datetimev2)) otherCondition=() -----filter((convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') > '2022-01-01 00:00:00')) +----filter((convert_tz(t1.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') > '2022-01-01 00:00:00')) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter((convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') > '2022-01-01 00:00:00')) +----filter((convert_tz(t2.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') > '2022-01-01 00:00:00')) ------PhysicalOlapScan[extend_infer_t2(t2)] -- !test_next_date_predicate -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_datetimev2 = t2.d_datetimev2)) otherCondition=() -----filter((dayofmonth(hours_add(convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) +----filter((dayofmonth(hours_add(convert_tz(t1.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter((dayofmonth(hours_add(convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) +----filter((dayofmonth(hours_add(convert_tz(t2.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) ------PhysicalOlapScan[extend_infer_t2(t2)] -- !test_random_nest_predicate -- PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.d_datetimev2 = t2.d_datetimev2)) otherCondition=((dayofmonth(hours_add(convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), cast(random(1, 10) as INT))) > 10)) +--hashJoin[INNER_JOIN] hashCondition=((t1.d_datetimev2 = t2.d_datetimev2)) otherCondition=((dayofmonth(hours_add(convert_tz(t1.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), cast(random(1, 10) as INT))) > 10)) ----PhysicalOlapScan[extend_infer_t1(t1)] ----PhysicalOlapScan[extend_infer_t2(t2)] -- !test_random_predicate -- PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.a = t2.a)) otherCondition=((cast(a as DOUBLE) > random(10))) +--hashJoin[INNER_JOIN] hashCondition=((t1.a = t2.a)) otherCondition=((cast(t1.a as DOUBLE) > random(10))) ----PhysicalOlapScan[extend_infer_t3(t1)] ----PhysicalOlapScan[extend_infer_t4(t2)] -- !test_predicate_map -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_datetimev2 = t2.d_datetimev2)) otherCondition=() -----filter((convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') < '2022-01-01 00:00:00') and (dayofmonth(hours_add(convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) +----filter((convert_tz(t1.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') < '2022-01-01 00:00:00') and (dayofmonth(hours_add(convert_tz(t1.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter((convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') < '2022-01-01 00:00:00') and (dayofmonth(hours_add(convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) +----filter((convert_tz(t2.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') < '2022-01-01 00:00:00') and (dayofmonth(hours_add(convert_tz(t2.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) ------PhysicalOlapScan[extend_infer_t2(t2)] -- !test_int_upcast -- @@ -235,7 +235,7 @@ PhysicalResultSink -- !test_int_downcast -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_int as TINYINT) = t2.d_tinyint)) otherCondition=() -----filter((cast(d_int as TINYINT) < 10)) +----filter((cast(t1.d_int as TINYINT) < 10)) ------PhysicalOlapScan[extend_infer_t1(t1)] ----filter((t2.d_tinyint < 10)) ------PhysicalOlapScan[extend_infer_t1(t2)] @@ -283,7 +283,7 @@ PhysicalResultSink -- !test_char_different_type2 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_substring(cast(d_char100 as CHAR(50)), 1, 50) = t2.d_char10)) otherCondition=() -----filter((substring(cast(d_char100 as CHAR(50)), 1, 50) > 'abc')) +----filter((substring(cast(t1.d_char100 as CHAR(50)), 1, 50) > 'abc')) ------PhysicalOlapScan[extend_infer_t1(t1)] ----filter((t2.d_char10 > 'abc')) ------PhysicalOlapScan[extend_infer_t1(t2)] @@ -312,7 +312,7 @@ PhysicalResultSink -- !test_cast_and_func2 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(abs(d_int) as TINYINT) = t2.d_tinyint)) otherCondition=() -----filter((cast(abs(d_int) as TINYINT) < 10)) +----filter((cast(abs(t1.d_int) as TINYINT) < 10)) ------PhysicalOlapScan[extend_infer_t1(t1)] ----filter((t2.d_tinyint < 10)) ------PhysicalOlapScan[extend_infer_t1(t2)] @@ -320,32 +320,32 @@ PhysicalResultSink -- !test_cast_and_func3 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(cast(d_int as TINYINT) as SMALLINT) = expr_abs(d_tinyint))) otherCondition=() -----filter((cast(d_int as TINYINT) < 10)) +----filter((cast(t1.d_int as TINYINT) < 10)) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter((abs(d_tinyint) < 10)) +----filter((abs(t2.d_tinyint) < 10)) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_cast_and_func4 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.d_int = expr_cast(abs(d_tinyint) as INT))) otherCondition=() ----PhysicalOlapScan[extend_infer_t1(t1)] -----filter((abs(d_tinyint) < 10)) +----filter((abs(t2.d_tinyint) < 10)) ------PhysicalOlapScan[extend_infer_t1(t2)] -- !test_func_equal_and_nest_func_pred1 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') = expr_convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'))) otherCondition=() -----filter((dayofmonth(hours_add(convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) +----filter((dayofmonth(hours_add(convert_tz(t1.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter((dayofmonth(hours_add(convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) +----filter((dayofmonth(hours_add(convert_tz(t2.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'), 10)) > 10)) ------PhysicalOlapScan[extend_infer_t2(t2)] -- !test_func_equal_and_nest_func_pred2 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris') = expr_convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris'))) otherCondition=() -----filter((dayofmonth(convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris')) > 10)) +----filter((dayofmonth(convert_tz(t1.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris')) > 10)) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter((dayofmonth(convert_tz(d_datetimev2, 'Asia/Shanghai', 'Europe/Paris')) > 10)) +----filter((dayofmonth(convert_tz(t2.d_datetimev2, 'Asia/Shanghai', 'Europe/Paris')) > 10)) ------PhysicalOlapScan[extend_infer_t2(t2)] -- !predicate_to_empty_relation -- @@ -495,169 +495,169 @@ PhysicalResultSink -- !not_equal_inner_left -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not (d_int = 10))) +----filter(( not (t3.d_int = 10))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((c1 = t2.d_int)) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_int = 10))) +------filter(( not (t2.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_equal_inner_left2 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not (d_int = 10))) +----filter(( not (t3.d_int = 10))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[INNER_JOIN] hashCondition=((t1.d_int = c1)) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_int = 10))) +------filter(( not (t2.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_equal_left_inner -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not (d_int = 10))) +----filter(( not (t3.d_int = 10))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[INNER_JOIN] hashCondition=((c1 = t2.d_int)) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_int = 10))) +------filter(( not (t2.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_equal_left_left -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not (d_int = 10))) +----filter(( not (t3.d_int = 10))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((c1 = t2.d_int)) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_int = 10))) +------filter(( not (t2.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_equal_left_left2 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not (d_int = 10))) +----filter(( not (t3.d_int = 10))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[INNER_JOIN] hashCondition=((t1.d_int = c1)) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_int = 10))) +------filter(( not (t2.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_in_inner_right -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not d_int IN (10, 20))) +----filter(( not t3.d_int IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[INNER_JOIN] hashCondition=((c1 = t2.d_int)) otherCondition=() -------filter(( not d_int IN (10, 20))) +------filter(( not t1.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not d_int IN (10, 20))) +------filter(( not t2.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_in_inner_right2 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not d_int IN (10, 20))) +----filter(( not t3.d_int IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.d_int = c1)) otherCondition=() -------filter(( not d_int IN (10, 20))) +------filter(( not t1.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not d_int IN (10, 20))) +------filter(( not t2.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_in_right_inner -- PhysicalResultSink --hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not d_int IN (10, 20))) +----filter(( not t3.d_int IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[INNER_JOIN] hashCondition=((c1 = t2.d_int)) otherCondition=() -------filter(( not d_int IN (10, 20))) +------filter(( not t1.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not d_int IN (10, 20))) +------filter(( not t2.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_in_right_right -- PhysicalResultSink --hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not d_int IN (10, 20))) +----filter(( not t3.d_int IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[INNER_JOIN] hashCondition=((c1 = t2.d_int)) otherCondition=() -------filter(( not d_int IN (10, 20))) +------filter(( not t1.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not d_int IN (10, 20))) +------filter(( not t2.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_in_right_right2 -- PhysicalResultSink --hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t3.d_int = t.c1)) otherCondition=() -----filter(( not d_int IN (10, 20))) +----filter(( not t3.d_int IN (10, 20))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.d_int = c1)) otherCondition=() -------filter(( not d_int IN (10, 20))) +------filter(( not t1.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not d_int IN (10, 20))) +------filter(( not t2.d_int IN (10, 20))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_equal_semi_semi_with_cast -- PhysicalResultSink --hashJoin[LEFT_SEMI_JOIN] hashCondition=((expr_cast(d_smallint as INT) = t.c1)) otherCondition=() -----filter(( not (d_smallint = 10))) +----filter(( not (t3.d_smallint = 10))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[LEFT_SEMI_JOIN] hashCondition=((c1 = expr_cast(d_tinyint as INT))) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_tinyint = 10))) +------filter(( not (t2.d_tinyint = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_equal_anti_anti_with_cast -- PhysicalResultSink --hashJoin[LEFT_ANTI_JOIN] hashCondition=((expr_cast(d_smallint as INT) = t.c1)) otherCondition=() -----filter(( not (d_smallint = 10))) +----filter(( not (t3.d_smallint = 10))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[LEFT_ANTI_JOIN] hashCondition=((c1 = expr_cast(d_tinyint as INT))) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_tinyint = 10))) +------filter(( not (t2.d_tinyint = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_equal_anti_left_with_cast -- PhysicalResultSink --hashJoin[LEFT_ANTI_JOIN] hashCondition=((expr_cast(d_smallint as INT) = t.c1)) otherCondition=() -----filter(( not (d_smallint = 10))) +----filter(( not (t3.d_smallint = 10))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((c1 = expr_cast(d_tinyint as INT))) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_tinyint = 10))) +------filter(( not (t2.d_tinyint = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !not_equal_semi_anti_with_cast -- PhysicalResultSink --hashJoin[LEFT_SEMI_JOIN] hashCondition=((expr_cast(d_smallint as INT) = t.c1)) otherCondition=() -----filter(( not (d_smallint = 10))) +----filter(( not (t3.d_smallint = 10))) ------PhysicalOlapScan[extend_infer_t1(t3)] ----hashJoin[LEFT_ANTI_JOIN] hashCondition=((c1 = expr_cast(d_tinyint as INT))) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_tinyint = 10))) +------filter(( not (t2.d_tinyint = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -- !in_subquery_to_semi_join -- PhysicalResultSink --hashJoin[LEFT_SEMI_JOIN] hashCondition=((t1.d_int = extend_infer_t2.d_int)) otherCondition=() -----filter(( not (d_int = 10))) +----filter(( not (t1.d_int = 10))) ------PhysicalOlapScan[extend_infer_t1(t1)] -----filter(( not (d_int = 10))) +----filter(( not (extend_infer_t2.d_int = 10))) ------PhysicalOlapScan[extend_infer_t2] -- !not_in_subquery_to_na_anti_join_not_infer -- PhysicalResultSink --hashJoin[NULL_AWARE_LEFT_ANTI_JOIN] hashCondition=((t1.d_int = extend_infer_t2.d_int)) otherCondition=() -----filter(( not (d_int = 10))) +----filter(( not (t1.d_int = 10))) ------PhysicalOlapScan[extend_infer_t1(t1)] ----PhysicalOlapScan[extend_infer_t2] @@ -665,31 +665,31 @@ PhysicalResultSink PhysicalResultSink --hashJoin[LEFT_SEMI_JOIN] hashCondition=((t1.d_int = extend_infer_t2.d_int)) otherCondition=() ----hashJoin[INNER_JOIN] hashCondition=((t1.d_int = t2.d_int)) otherCondition=() -------filter(( not (d_int = 10))) +------filter(( not (t1.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t1)] -------filter(( not (d_int = 10))) +------filter(( not (t2.d_int = 10))) --------PhysicalOlapScan[extend_infer_t1(t2)] -----filter(( not (d_int = 10))) +----filter(( not (extend_infer_t2.d_int = 10))) ------PhysicalOlapScan[extend_infer_t2] -- !cast_to_decimal_overflow_not_infer -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_tinyint as INT) = t2.d_int)) otherCondition=() -----filter(cast(d_tinyint as DECIMALV3(4, 1)) IN (0.1, 0.5)) +----filter(cast(t1.d_tinyint as DECIMALV3(4, 1)) IN (0.1, 0.5)) ------PhysicalOlapScan[extend_infer_t1(t1)] ----PhysicalOlapScan[extend_infer_t2(t2)] -- !char_equal_int_infer -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_char10 as DECIMALV3(38, 6)) = expr_cast(d_int as DECIMALV3(38, 6)))) otherCondition=() -----filter(d_char10 IN ('bb', 'd')) +----filter(t1.d_char10 IN ('bb', 'd')) ------PhysicalOlapScan[extend_infer_t1(t1)] ----PhysicalOlapScan[extend_infer_t2(t2)] -- !date_equal_int_infer -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_datev2 as DATETIMEV2(0)) = expr_cast(d_int as DATETIMEV2(0)))) otherCondition=() -----filter(d_datev2 IN ('2024-01-01', '2024-01-02')) +----filter(t1.d_datev2 IN ('2024-01-01', '2024-01-02')) ------PhysicalOlapScan[extend_infer_t1(t1)] ----PhysicalOlapScan[extend_infer_t2(t2)] @@ -808,11 +808,11 @@ PhysicalResultSink PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((t2.a = t3.a)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.a = t2.a)) otherCondition=() -------filter(a IN (1, 2)) +------filter(t1.a IN (1, 2)) --------PhysicalOlapScan[extend_infer_t3(t1)] -------filter(a IN (1, 2)) +------filter(t2.a IN (1, 2)) --------PhysicalOlapScan[extend_infer_t4(t2)] -----filter(a IN (1, 2)) +----filter(t3.a IN (1, 2)) ------PhysicalOlapScan[extend_infer_t5(t3)] -- !qt_leftjoin_right_pull_up_in_shape_result -- @@ -821,7 +821,7 @@ PhysicalResultSink PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((t2.a = t3.a)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.a = t2.a)) otherCondition=() -------filter(a IS NULL) +------filter(t1.a IS NULL) --------PhysicalOlapScan[extend_infer_t3(t1)] ------PhysicalOlapScan[extend_infer_t4(t2)] ----PhysicalOlapScan[extend_infer_t5(t3)] @@ -834,7 +834,7 @@ PhysicalResultSink PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((t2.a = t3.a)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.a = t2.a)) otherCondition=() -------filter(( not a IS NULL)) +------filter(( not t1.a IS NULL)) --------PhysicalOlapScan[extend_infer_t3(t1)] ------PhysicalOlapScan[extend_infer_t4(t2)] ----PhysicalOlapScan[extend_infer_t5(t3)] diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/infer_intersect_except.out b/regression-test/data/nereids_rules_p0/infer_predicate/infer_intersect_except.out index 8ba73f974738d7..a7411cbdc9ce46 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/infer_intersect_except.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/infer_intersect_except.out @@ -60,34 +60,34 @@ PhysicalResultSink ----PhysicalIntersect ------filter((infer_intersect_except2.a < 10) and (infer_intersect_except2.a > 0) and (infer_intersect_except2.b > 'ab')) --------PhysicalOlapScan[infer_intersect_except2] -------filter((cast(b as TEXT) = 'abc') and (infer_intersect_except3.a < 10) and (infer_intersect_except3.a > 0)) +------filter((cast(infer_intersect_except3.b as TEXT) = 'abc') and (infer_intersect_except3.a < 10) and (infer_intersect_except3.a > 0)) --------PhysicalOlapScan[infer_intersect_except3] -- !intersect_and_except -- PhysicalResultSink --PhysicalExcept ----PhysicalIntersect -------filter((cast(b as TEXT) = 'abc') and (infer_intersect_except1.a = 1)) +------filter((cast(infer_intersect_except1.b as TEXT) = 'abc') and (infer_intersect_except1.a = 1)) --------PhysicalOlapScan[infer_intersect_except1] ------filter((infer_intersect_except2.b > 'ab')) --------PhysicalOlapScan[infer_intersect_except2] -----filter((cast(b as TEXT) = 'abc') and (infer_intersect_except3.a = 1)) +----filter((cast(infer_intersect_except3.b as TEXT) = 'abc') and (infer_intersect_except3.a = 1)) ------PhysicalOlapScan[infer_intersect_except3] -- !function_intersect -- PhysicalResultSink --PhysicalIntersect -----filter((abs(a) < 3)) +----filter((abs(t1.a) < 3)) ------PhysicalOlapScan[infer_intersect_except1(t1)] -----filter((abs(a) < 3)) +----filter((abs(t2.a) < 3)) ------PhysicalOlapScan[infer_intersect_except2(t2)] -- !function_except -- PhysicalResultSink --PhysicalExcept -----filter((abs(a) < 3)) +----filter((abs(t1.a) < 3)) ------PhysicalOlapScan[infer_intersect_except1(t1)] -----filter((abs(a) < 3)) +----filter((abs(t2.a) < 3)) ------PhysicalOlapScan[infer_intersect_except2(t2)] -- !except_res -- diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/infer_unequal_predicates.out b/regression-test/data/nereids_rules_p0/infer_predicate/infer_unequal_predicates.out index 07ecdfb27a157c..8d5b401f4c1fd9 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/infer_unequal_predicates.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/infer_unequal_predicates.out @@ -136,14 +136,14 @@ PhysicalResultSink -- !expr_unequal_infer_same_table2 -- PhysicalResultSink ---filter((abs(c) < 1) and (abs(d) < abs(c))) +--filter((abs(t1.c) < 1) and (abs(t1.d) < abs(t1.c))) ----PhysicalOlapScan[infer_unequal_predicates_t1(t1)] -- !expr_unequal_infer_diff_table -- PhysicalResultSink --NestedLoopJoin[INNER_JOIN](abs(d) < abs(c)) ----PhysicalOlapScan[infer_unequal_predicates_t1(t1)] -----filter((abs(c) < 1)) +----filter((abs(t2.c) < 1)) ------PhysicalOlapScan[infer_unequal_predicates_t2(t2)] -- !not_infer_expr1 -- diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out index 0fdfa46eae00bb..e352e38da3ae18 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out @@ -161,7 +161,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_bool as TINYINT) = 127)) +--------filter((cast(t2.d_bool as TINYINT) = 127)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type6 -- @@ -233,7 +233,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_char as DECIMALV3(38, 6)) = 127.000000)) +--------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 127.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type12 -- @@ -245,7 +245,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_varchar as DECIMALV3(38, 6)) = 127.000000)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 127.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type13 -- @@ -257,7 +257,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_string as DECIMALV3(38, 6)) = 127.000000)) +--------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 127.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type16 -- @@ -321,7 +321,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_bool as SMALLINT) = 32767)) +--------filter((cast(t2.d_bool as SMALLINT) = 32767)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type22 -- @@ -377,7 +377,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_char as DECIMALV3(38, 6)) = 32767.000000)) +--------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 32767.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type28 -- @@ -389,7 +389,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_varchar as DECIMALV3(38, 6)) = 32767.000000)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 32767.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type29 -- @@ -401,7 +401,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_string as DECIMALV3(38, 6)) = 32767.000000)) +--------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 32767.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type32 -- @@ -457,7 +457,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_bool as INT) = 32768)) +--------filter((cast(t2.d_bool as INT) = 32768)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type38 -- @@ -469,7 +469,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_float as DOUBLE) = 32768.0)) +--------filter((cast(t2.d_float as DOUBLE) = 32768.0)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type39 -- @@ -513,7 +513,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_char as DECIMALV3(38, 6)) = 32768.000000)) +--------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 32768.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type44 -- @@ -525,7 +525,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_varchar as DECIMALV3(38, 6)) = 32768.000000)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 32768.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type45 -- @@ -537,7 +537,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_string as DECIMALV3(38, 6)) = 32768.000000)) +--------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 32768.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type48 -- @@ -585,7 +585,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_bool as BIGINT) = 214748364799)) +--------filter((cast(t2.d_bool as BIGINT) = 214748364799)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type54 -- @@ -597,7 +597,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_float as DOUBLE) = 2.14748364799E11)) +--------filter((cast(t2.d_float as DOUBLE) = 2.14748364799E11)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type55 -- @@ -621,7 +621,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_decimal as DECIMALV3(38, 18)) = 214748364799.000000000000000000)) +--------filter((cast(t2.d_decimal as DECIMALV3(38, 18)) = 214748364799.000000000000000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type57 -- @@ -641,7 +641,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_char as DECIMALV3(38, 6)) = 214748364799.000000)) +--------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 214748364799.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type60 -- @@ -653,7 +653,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_varchar as DECIMALV3(38, 6)) = 214748364799.000000)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 214748364799.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type61 -- @@ -665,7 +665,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_string as DECIMALV3(38, 6)) = 214748364799.000000)) +--------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 214748364799.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type64 -- @@ -705,7 +705,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_bool as LARGEINT) = 922337203685477580722)) +--------filter((cast(t2.d_bool as LARGEINT) = 922337203685477580722)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type70 -- @@ -717,7 +717,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_float as DOUBLE) = 9.223372036854776E20)) +--------filter((cast(t2.d_float as DOUBLE) = 9.223372036854776E20)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type71 -- @@ -753,7 +753,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_char as DECIMALV3(38, 6)) = 922337203685477580722.000000)) +--------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 922337203685477580722.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type76 -- @@ -765,7 +765,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_varchar as DECIMALV3(38, 6)) = 922337203685477580722.000000)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 922337203685477580722.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type77 -- @@ -777,7 +777,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_string as DECIMALV3(38, 6)) = 922337203685477580722.000000)) +--------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 922337203685477580722.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type80 -- @@ -897,7 +897,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter(cast(d_char as BOOLEAN)) +--------filter(cast(t2.d_char as BOOLEAN)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type92 -- @@ -909,7 +909,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter(cast(d_varchar as BOOLEAN)) +--------filter(cast(t2.d_varchar as BOOLEAN)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type93 -- @@ -921,7 +921,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter(cast(d_string as BOOLEAN)) +--------filter(cast(t2.d_string as BOOLEAN)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type96 -- @@ -949,7 +949,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_largeint as DECIMALV3(38, 4)) = 1.0001)) +--------filter((cast(t2.d_largeint as DECIMALV3(38, 4)) = 1.0001)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type101 -- @@ -961,7 +961,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_bool as DECIMALV3(5, 4)) = 1.0001)) +--------filter((cast(t2.d_bool as DECIMALV3(5, 4)) = 1.0001)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type102 -- @@ -973,7 +973,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_float as DOUBLE) = 1.0001)) +--------filter((cast(t2.d_float as DOUBLE) = 1.0001)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type103 -- @@ -1017,7 +1017,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_char as DECIMALV3(38, 6)) = 1.000100)) +--------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 1.000100)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type108 -- @@ -1029,7 +1029,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_varchar as DECIMALV3(38, 6)) = 1.000100)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 1.000100)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type109 -- @@ -1041,7 +1041,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_string as DECIMALV3(38, 6)) = 1.000100)) +--------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 1.000100)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type112 -- @@ -1069,7 +1069,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_largeint as DECIMALV3(38, 6)) = 1.000000)) +--------filter((cast(t2.d_largeint as DECIMALV3(38, 6)) = 1.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type117 -- @@ -1081,7 +1081,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_bool as DECIMALV3(13, 12)) = 1.000000000003)) +--------filter((cast(t2.d_bool as DECIMALV3(13, 12)) = 1.000000000003)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type118 -- @@ -1093,7 +1093,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_float as DOUBLE) = 1.000000000003)) +--------filter((cast(t2.d_float as DOUBLE) = 1.000000000003)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type119 -- @@ -1137,7 +1137,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_char as DECIMALV3(38, 6)) = 1.000000)) +--------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 1.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type124 -- @@ -1149,7 +1149,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_varchar as DECIMALV3(38, 6)) = 1.000000)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 1.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type125 -- @@ -1161,7 +1161,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_string as DECIMALV3(38, 6)) = 1.000000)) +--------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 1.000000)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type128 -- @@ -1189,7 +1189,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_largeint as DECIMALV3(38, 6)) = 12232.239827)) +--------filter((cast(t2.d_largeint as DECIMALV3(38, 6)) = 12232.239827)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type133 -- @@ -1201,7 +1201,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_bool as DECIMALV3(21, 16)) = 12232.2398272342335234)) +--------filter((cast(t2.d_bool as DECIMALV3(21, 16)) = 12232.2398272342335234)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type134 -- @@ -1213,7 +1213,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_float as DOUBLE) = 12232.239827234234)) +--------filter((cast(t2.d_float as DOUBLE) = 12232.239827234234)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type135 -- @@ -1257,7 +1257,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_char as DECIMALV3(38, 6)) = 12232.239827)) +--------filter((cast(t2.d_char as DECIMALV3(38, 6)) = 12232.239827)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type140 -- @@ -1269,7 +1269,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_varchar as DECIMALV3(38, 6)) = 12232.239827)) +--------filter((cast(t2.d_varchar as DECIMALV3(38, 6)) = 12232.239827)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type141 -- @@ -1281,7 +1281,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((cast(d_string as DECIMALV3(38, 6)) = 12232.239827)) +--------filter((cast(t2.d_string as DECIMALV3(38, 6)) = 12232.239827)) ----------PhysicalOlapScan[test_types(t2)] -- !const_value_and_join_column_type144 -- diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.out b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.out index cd6b1b43c11747..c12f2c2c4b0757 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_set_op.out @@ -127,14 +127,14 @@ PhysicalResultSink PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.a = t.a) and (t3.b = t.b)) otherCondition=() ----PhysicalUnion -----filter(a IN (133333, 2) and b IN ('aa', 'dd')) +----filter(t3.a IN (133333, 2) and t3.b IN ('aa', 'dd')) ------PhysicalOlapScan[test_pull_up_predicate_set_op3(t3)] -- !union_all_const_tinyint_int -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.a = expr_cast(a as INT)) and (t3.b = t.b)) otherCondition=() ----PhysicalUnion -----filter(a IN (2, 3) and b IN ('aa', 'dd')) +----filter(t3.a IN (2, 3) and t3.b IN ('aa', 'dd')) ------PhysicalOlapScan[test_pull_up_predicate_set_op3(t3)] -- !union_all_const_empty_relation -- @@ -148,28 +148,28 @@ PhysicalResultSink PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(a as DECIMALV3(38, 6)) = expr_cast(a as DECIMALV3(38, 6))) and (t3.b = t.b)) otherCondition=() ----PhysicalUnion -----filter(b IN ('aa', 'dd')) +----filter(t3.b IN ('aa', 'dd')) ------PhysicalOlapScan[test_pull_up_predicate_set_op3(t3)] -- !union_all_const2_has_cast_to_null_different_type -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(a as DECIMALV3(38, 6)) = expr_cast(a as DECIMALV3(38, 6))) and (t3.b = t.b)) otherCondition=() ----PhysicalUnion -----filter(b IN ('12', 'dd')) +----filter(t3.b IN ('12', 'dd')) ------PhysicalOlapScan[test_pull_up_predicate_set_op3(t3)] -- !union_all_different_type_int_cast_to_char -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(a as DECIMALV3(38, 6)) = expr_cast(a as DECIMALV3(38, 6))) and (t3.b = t.b)) otherCondition=() ----PhysicalUnion -----filter(b IN ('12', 'dd')) +----filter(t3.b IN ('12', 'dd')) ------PhysicalOlapScan[test_pull_up_predicate_set_op3(t3)] -- !union_all_const_char3_char2 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.a = expr_cast(a as INT)) and (t3.b = t.b)) otherCondition=() ----PhysicalUnion -----filter(a IN (2, 3, 5) and b IN ('aa', 'aab', 'dd')) +----filter(t3.a IN (2, 3, 5) and t3.b IN ('aa', 'aab', 'dd')) ------PhysicalOlapScan[test_pull_up_predicate_set_op3(t3)] -- !union_child_and_const_exprs -- @@ -178,11 +178,11 @@ PhysicalResultSink ----hashAgg[GLOBAL] ------hashAgg[LOCAL] --------PhysicalUnion -----------filter(a IN (1, 2)) +----------filter(test_pull_up_predicate_set_op1.a IN (1, 2)) ------------PhysicalOlapScan[test_pull_up_predicate_set_op1] -----------filter(a IN (1, 2)) +----------filter(test_pull_up_predicate_set_op2.a IN (1, 2)) ------------PhysicalOlapScan[test_pull_up_predicate_set_op2] -----filter(a IN (1, 2)) +----filter(t3.a IN (1, 2)) ------PhysicalOlapScan[test_pull_up_predicate_set_op3(t3)] -- !union_child_and_const_exprs_andpredicates -- @@ -191,9 +191,9 @@ PhysicalResultSink ----hashAgg[GLOBAL] ------hashAgg[LOCAL] --------PhysicalUnion -----------filter(a IN (1, 2) and b IN ('2d', '3')) +----------filter(test_pull_up_predicate_set_op1.a IN (1, 2) and test_pull_up_predicate_set_op1.b IN ('2d', '3')) ------------PhysicalOlapScan[test_pull_up_predicate_set_op1] -----filter(a IN (1, 2)) +----filter(t3.a IN (1, 2)) ------PhysicalOlapScan[test_pull_up_predicate_set_op3(t3)] -- !union_child_and_const_exprs_orpredicates -- @@ -202,7 +202,7 @@ PhysicalResultSink ----hashAgg[GLOBAL] ------hashAgg[LOCAL] --------PhysicalUnion -----------filter(OR[a IN (1, 2),b IN ('2d', '3')]) +----------filter(OR[test_pull_up_predicate_set_op1.a IN (1, 2),test_pull_up_predicate_set_op1.b IN ('2d', '3')]) ------------PhysicalOlapScan[test_pull_up_predicate_set_op1] ----PhysicalOlapScan[test_pull_up_predicate_set_op3(t3)] @@ -374,28 +374,28 @@ PhysicalResultSink PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_datetimev2 as DATETIMEV2(6)) = expr_cast(b as DATETIMEV2(6))) and (expr_cast(d_smallint as INT) = t.a)) otherCondition=() ----PhysicalUnion -----filter(cast(d_smallint as INT) IN (12222222, 2)) +----filter(cast(t3.d_smallint as INT) IN (12222222, 2)) ------PhysicalOlapScan[test_pull_up_predicate_set_op4(t3)] -- !union_all_const_date -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_datev2 as DATETIMEV2(6)) = expr_cast(b as DATETIMEV2(6))) and (expr_cast(d_smallint as INT) = t.a)) otherCondition=() ----PhysicalUnion -----filter(cast(d_smallint as INT) IN (12222222, 2)) +----filter(cast(t3.d_smallint as INT) IN (12222222, 2)) ------PhysicalOlapScan[test_pull_up_predicate_set_op4(t3)] -- !union_all_const_char100 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((expr_cast(d_smallint as INT) = t.a) and (t3.d_char100 = t.b)) otherCondition=() ----PhysicalUnion -----filter(cast(d_smallint as INT) IN (12222222, 2) and d_char100 IN ('2024-01-03 10:00:00', '2024-01-03')) +----filter(cast(t3.d_smallint as INT) IN (12222222, 2) and t3.d_char100 IN ('2024-01-03 10:00:00', '2024-01-03')) ------PhysicalOlapScan[test_pull_up_predicate_set_op4(t3)] -- !union_all_const_char10 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t3.d_char10 = t.b)) otherCondition=() ----PhysicalUnion -----filter(d_char10 IN ('2024-01-03 10:00:00', '2024-01-04')) +----filter(t3.d_char10 IN ('2024-01-03 10:00:00', '2024-01-04')) ------PhysicalOlapScan[test_pull_up_predicate_set_op4(t3)] -- !union_all_and_project_pull_up -- @@ -413,7 +413,7 @@ PhysicalResultSink ----hashAgg[GLOBAL] ------hashAgg[LOCAL] --------PhysicalUnion -----filter(a IN (1, 2)) +----filter(t2.a IN (1, 2)) ------PhysicalOlapScan[test_pull_up_predicate_set_op3(t2)] -- !intersect_res -- diff --git a/regression-test/data/nereids_rules_p0/join_extract_or_from_case_when/join_extract_or_from_case_when.out b/regression-test/data/nereids_rules_p0/join_extract_or_from_case_when/join_extract_or_from_case_when.out index 84590c64610e4d..ccfad2036b655c 100644 --- a/regression-test/data/nereids_rules_p0/join_extract_or_from_case_when/join_extract_or_from_case_when.out +++ b/regression-test/data/nereids_rules_p0/join_extract_or_from_case_when/join_extract_or_from_case_when.out @@ -2,57 +2,57 @@ -- !case_when_one_side_1 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN]( not (CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) END = 95)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a] ---------filter(OR[( not (a = 95)),( not ((a + b) = 95))]) +----NestedLoopJoin[INNER_JOIN]( not (CASE WHEN (x > 10) THEN t1.a WHEN (y > 10) THEN (a + b) END = 95)) +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a] +--------filter(OR[( not (t1.a = 95)),( not ((t1.a + t1.b) = 95))]) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[(t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_one_side_2 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN]( not (CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) END = ((a - b) - 5))) -------PhysicalProject[((a - b) - 5) AS `((a - b) - 5)`, (a + b) AS `(a + b)`, t1.a] ---------filter(OR[( not (a = ((a - b) - 5))),( not ((a + b) = ((a - b) - 5)))]) +----NestedLoopJoin[INNER_JOIN]( not (CASE WHEN (x > 10) THEN t1.a WHEN (y > 10) THEN (a + b) END = ((a - b) - 5))) +------PhysicalProject[((t1.a - t1.b) - 5) AS `((a - b) - 5)`, (t1.a + t1.b) AS `(a + b)`, t1.a] +--------filter(OR[( not (t1.a = ((t1.a - t1.b) - 5))),( not ((t1.a + t1.b) = ((t1.a - t1.b) - 5)))]) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[(t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_one_side_3 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN]( not (CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE b END = 95)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a, t1.b] ---------filter(OR[( not (a = 95)),( not ((a + b) = 95)),( not (b = 95))]) +----NestedLoopJoin[INNER_JOIN]( not (CASE WHEN (x > 10) THEN t1.a WHEN (y > 10) THEN (a + b) ELSE t1.b END = 95)) +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a, t1.b] +--------filter(OR[( not (t1.a = 95)),( not ((t1.a + t1.b) = 95)),( not (t1.b = 95))]) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[(t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_one_side_4 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN]( not (CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE b END = ((a - b) - 5))) -------PhysicalProject[((a - b) - 5) AS `((a - b) - 5)`, (a + b) AS `(a + b)`, t1.a, t1.b] ---------filter(OR[( not (a = ((a - b) - 5))),( not ((a + b) = ((a - b) - 5))),( not (b = ((a - b) - 5)))]) +----NestedLoopJoin[INNER_JOIN]( not (CASE WHEN (x > 10) THEN t1.a WHEN (y > 10) THEN (a + b) ELSE t1.b END = ((a - b) - 5))) +------PhysicalProject[((t1.a - t1.b) - 5) AS `((a - b) - 5)`, (t1.a + t1.b) AS `(a + b)`, t1.a, t1.b] +--------filter(OR[( not (t1.a = ((t1.a - t1.b) - 5))),( not ((t1.a + t1.b) = ((t1.a - t1.b) - 5))),( not (t1.b = ((t1.a - t1.b) - 5)))]) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[(t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_one_side_5 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN]((CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE b END + random(1, 100)) = (a - b)) -------PhysicalProject[(a + b) AS `(a + b)`, (a - b) AS `(a - b)`, t1.a, t1.b] +----NestedLoopJoin[INNER_JOIN]((CASE WHEN (x > 10) THEN t1.a WHEN (y > 10) THEN (a + b) ELSE t1.b END + random(1, 100)) = (a - b)) +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, (t1.a - t1.b) AS `(a - b)`, t1.a, t1.b] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[(t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_one_side_6 -- PhysicalResultSink --NestedLoopJoin[CROSS_JOIN] ----PhysicalProject[t1.a] -------filter(( not (CASE WHEN (a > 10) THEN a WHEN (b > 10) THEN (a + b) END = 95))) +------filter(( not (CASE WHEN (t1.a > 10) THEN t1.a WHEN (t1.b > 10) THEN (t1.a + t1.b) END = 95))) --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] ----PhysicalProject[t2.x] ------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] @@ -60,147 +60,147 @@ PhysicalResultSink -- !case_when_one_side_7 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN]((CASE WHEN (x > 0) THEN a WHEN (x < 10) THEN (a + 1) END + CASE WHEN (x > 1) THEN (a + 1) WHEN (x < 10) THEN (a + 10) END) > (a + b)) -------PhysicalProject[(a + 1) AS `(a + 1)`, (a + 10) AS `(a + 10)`, (a + b) AS `(a + b)`, t1.a] +----NestedLoopJoin[INNER_JOIN]((CASE WHEN (x > 0) THEN t1.a WHEN (x < 10) THEN (a + 1) END + CASE WHEN (x > 1) THEN (a + 1) WHEN (x < 10) THEN (a + 10) END) > (a + b)) +------PhysicalProject[(t1.a + 1) AS `(a + 1)`, (t1.a + 10) AS `(a + 10)`, (t1.a + t1.b) AS `(a + b)`, t1.a] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x < 10) AS `(x < 10)`, (x > 0) AS `(x > 0)`, (x > 1) AS `(x > 1)`, t2.x] +------PhysicalProject[(t2.x < 10) AS `(x < 10)`, (t2.x > 0) AS `(x > 0)`, (t2.x > 1) AS `(x > 1)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_one_side_8 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN]((CASE WHEN (x > 0) THEN a WHEN (x < 10) THEN (a + 1) END + CASE WHEN (a > 1) THEN (a + 1) WHEN (a < 10) THEN (a + 10) END) > (a + b)) -------PhysicalProject[(a + 1) AS `(a + 1)`, (a + b) AS `(a + b)`, CASE WHEN (a > 1) THEN (a + 1) WHEN (a < 10) THEN (a + 10) END AS `CASE WHEN (a > 1) THEN (a + 1) WHEN (a < 10) THEN (a + 10) END`, t1.a] ---------filter(OR[((t1.a + CASE WHEN (a > 1) THEN (a + 1) WHEN (a < 10) THEN (a + 10) END) > (t1.a + t1.b)),((t1.a + CASE WHEN (a > 1) THEN (a + 1) WHEN (a < 10) THEN (a + 10) END) > ((t1.a + t1.b) - 1))]) +----NestedLoopJoin[INNER_JOIN]((CASE WHEN (x > 0) THEN t1.a WHEN (x < 10) THEN (a + 1) END + CASE WHEN (a > 1) THEN (a + 1) WHEN (a < 10) THEN (a + 10) END) > (a + b)) +------PhysicalProject[(t1.a + 1) AS `(a + 1)`, (t1.a + t1.b) AS `(a + b)`, CASE WHEN (t1.a > 1) THEN (t1.a + 1) WHEN (t1.a < 10) THEN (t1.a + 10) END AS `CASE WHEN (a > 1) THEN (a + 1) WHEN (a < 10) THEN (a + 10) END`, t1.a] +--------filter(OR[((t1.a + CASE WHEN (t1.a > 1) THEN (t1.a + 1) WHEN (t1.a < 10) THEN (t1.a + 10) END) > (t1.a + t1.b)),((t1.a + CASE WHEN (t1.a > 1) THEN (t1.a + 1) WHEN (t1.a < 10) THEN (t1.a + 10) END) > ((t1.a + t1.b) - 1))]) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x < 10) AS `(x < 10)`, (x > 0) AS `(x > 0)`, t2.x] +------PhysicalProject[(t2.x < 10) AS `(x < 10)`, (t2.x > 0) AS `(x > 0)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_one_side_9 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] ----NestedLoopJoin[INNER_JOIN](CASE WHEN (x > 0) THEN 105 WHEN (x < 10) THEN 10005 ELSE NULL END > (a + b)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a] +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a] --------filter(((t1.a + t1.b) < 10005)) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[CASE WHEN (x > 0) THEN 105 WHEN (x < 10) THEN 10005 ELSE NULL END AS `CASE WHEN (x > 0) THEN 105 WHEN (x < 10) THEN 10005 ELSE NULL END`, t2.x] +------PhysicalProject[CASE WHEN (t2.x > 0) THEN 105 WHEN (t2.x < 10) THEN 10005 ELSE NULL END AS `CASE WHEN (x > 0) THEN 105 WHEN (x < 10) THEN 10005 ELSE NULL END`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_one_side_10 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] ----NestedLoopJoin[LEFT_OUTER_JOIN](CASE WHEN (x > 0) THEN 105 WHEN (x < 10) THEN 10005 ELSE NULL END > (a + b)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a] +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[CASE WHEN (x > 0) THEN 105 WHEN (x < 10) THEN 10005 ELSE NULL END AS `CASE WHEN (x > 0) THEN 105 WHEN (x < 10) THEN 10005 ELSE NULL END`, t2.x] +------PhysicalProject[CASE WHEN (t2.x > 0) THEN 105 WHEN (t2.x < 10) THEN 10005 ELSE NULL END AS `CASE WHEN (x > 0) THEN 105 WHEN (x < 10) THEN 10005 ELSE NULL END`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_one_side_11 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] ----NestedLoopJoin[RIGHT_OUTER_JOIN](CASE WHEN (a > 0) THEN 105 WHEN (b < 10) THEN 10005 ELSE NULL END > (x + y)) -------PhysicalProject[CASE WHEN (a > 0) THEN 105 WHEN (b < 10) THEN 10005 ELSE NULL END AS `CASE WHEN (a > 0) THEN 105 WHEN (b < 10) THEN 10005 ELSE NULL END`, t1.a] +------PhysicalProject[CASE WHEN (t1.a > 0) THEN 105 WHEN (t1.b < 10) THEN 10005 ELSE NULL END AS `CASE WHEN (a > 0) THEN 105 WHEN (b < 10) THEN 10005 ELSE NULL END`, t1.a] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x + y) AS `(x + y)`, t2.x] +------PhysicalProject[(t2.x + t2.y) AS `(x + y)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !hash_cond_for_one_side_1 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] ----hashJoin[INNER_JOIN] hashCondition=((expr_if((a > 1), 1, 100) = expr_if((x > 2), 1, 200))) otherCondition=() -------PhysicalProject[if((a > 1), 1, 100) AS `expr_if((a > 1), 1, 100)`, t1.a] +------PhysicalProject[if((t1.a > 1), 1, 100) AS `expr_if((a > 1), 1, 100)`, t1.a] --------filter((t1.a > 1)) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[if((x > 2), 1, 200) AS `expr_if((x > 2), 1, 200)`, t2.x] +------PhysicalProject[if((t2.x > 2), 1, 200) AS `expr_if((x > 2), 1, 200)`, t2.x] --------filter((t2.x > 2)) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_two_side_1 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject[(a + b) AS `(a + b)`, t1.a, t1.b] +----PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a, t1.b] ------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject[((x + y) + -4) AS `((x + y) + -4)`, (x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[((t2.x + t2.y) + -4) AS `((x + y) + -4)`, (t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] ----PhysicalResultSink ------PhysicalUnion --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.a = .((x + y) + -4))) otherCondition=((CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE b END = .((x + y) + -4))) +----------hashJoin[INNER_JOIN] hashCondition=((.a = .((x + y) + -4))) otherCondition=((CASE WHEN .(x > 10) THEN .a WHEN .(y > 10) THEN .(a + b) ELSE .b END = .((x + y) + -4))) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.(a + b) = .((x + y) + -4))) otherCondition=((CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE b END = .((x + y) + -4)) and OR[( not (a = ((x + y) + -4))),(a = ((x + y) + -4)) IS NULL]) +----------hashJoin[INNER_JOIN] hashCondition=((.(a + b) = .((x + y) + -4))) otherCondition=((CASE WHEN .(x > 10) THEN .a WHEN .(y > 10) THEN .(a + b) ELSE .b END = .((x + y) + -4)) and OR[( not (.a = .((x + y) + -4))),(.a = .((x + y) + -4)) IS NULL]) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.b = .((x + y) + -4))) otherCondition=((CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE b END = .((x + y) + -4)) and OR[( not ((a + b) = ((x + y) + -4))),((a + b) = ((x + y) + -4)) IS NULL] and OR[( not (a = ((x + y) + -4))),(a = ((x + y) + -4)) IS NULL]) +----------hashJoin[INNER_JOIN] hashCondition=((.b = .((x + y) + -4))) otherCondition=((CASE WHEN .(x > 10) THEN .a WHEN .(y > 10) THEN .(a + b) ELSE .b END = .((x + y) + -4)) and OR[( not (.(a + b) = .((x + y) + -4))),(.(a + b) = .((x + y) + -4)) IS NULL] and OR[( not (.a = .((x + y) + -4))),(.a = .((x + y) + -4)) IS NULL]) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) -- !case_when_two_side_2 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject[(a + b) AS `(a + b)`, t1.a] +----PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a] ------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject[((x + y) + -4) AS `((x + y) + -4)`, (x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[((t2.x + t2.y) + -4) AS `((x + y) + -4)`, (t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] ----PhysicalResultSink ------PhysicalUnion --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.a = .((x + y) + -4))) otherCondition=((CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) END = .((x + y) + -4))) +----------hashJoin[INNER_JOIN] hashCondition=((.a = .((x + y) + -4))) otherCondition=((CASE WHEN .(x > 10) THEN .a WHEN .(y > 10) THEN .(a + b) END = .((x + y) + -4))) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.(a + b) = .((x + y) + -4))) otherCondition=((CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) END = .((x + y) + -4)) and OR[( not (a = ((x + y) + -4))),(a = ((x + y) + -4)) IS NULL]) +----------hashJoin[INNER_JOIN] hashCondition=((.(a + b) = .((x + y) + -4))) otherCondition=((CASE WHEN .(x > 10) THEN .a WHEN .(y > 10) THEN .(a + b) END = .((x + y) + -4)) and OR[( not (.a = .((x + y) + -4))),(.a = .((x + y) + -4)) IS NULL]) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) -- !case_when_two_side_3 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN]((CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE b END + random(1, 10)) = ((x + y) + 1)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a, t1.b] +----NestedLoopJoin[INNER_JOIN]((CASE WHEN (x > 10) THEN t1.a WHEN (y > 10) THEN (a + b) ELSE t1.b END + random(1, 10)) = ((x + y) + 1)) +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a, t1.b] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[((x + y) + 1) AS `((x + y) + 1)`, (x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[((t2.x + t2.y) + 1) AS `((x + y) + 1)`, (t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_two_side_4 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN](CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE (x + b) END = ((x + y) + 1)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a, t1.b] +----NestedLoopJoin[INNER_JOIN](CASE WHEN (x > 10) THEN t1.a WHEN (y > 10) THEN (a + b) ELSE (t2.x + t1.b) END = ((x + y) + 1)) +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a, t1.b] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[((x + y) + 1) AS `((x + y) + 1)`, (x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[((t2.x + t2.y) + 1) AS `((x + y) + 1)`, (t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !case_when_two_side_5 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject[(a + b) AS `(a + b)`, t1.a, t1.b] +----PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a, t1.b] ------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject[((x + y) + -4) AS `((x + y) + -4)`, (x - y) AS `(x - y)`, (x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[((t2.x + t2.y) + -4) AS `((x + y) + -4)`, (t2.x - t2.y) AS `(x - y)`, (t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] ----PhysicalResultSink ------PhysicalUnion --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.a = .(x - y))) otherCondition=((CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE b END = .((x + y) + -4)) and (if((a > x), a, b) = .(x - y))) +----------hashJoin[INNER_JOIN] hashCondition=((.a = .(x - y))) otherCondition=((CASE WHEN .(x > 10) THEN .a WHEN .(y > 10) THEN .(a + b) ELSE .b END = .((x + y) + -4)) and (if((.a > .x), .a, .b) = .(x - y))) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.b = .(x - y))) otherCondition=((CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE b END = .((x + y) + -4)) and (if((a > x), a, b) = .(x - y)) and OR[( not (a = (x - y))),(a = (x - y)) IS NULL]) +----------hashJoin[INNER_JOIN] hashCondition=((.b = .(x - y))) otherCondition=((CASE WHEN .(x > 10) THEN .a WHEN .(y > 10) THEN .(a + b) ELSE .b END = .((x + y) + -4)) and (if((.a > .x), .a, .b) = .(x - y)) and OR[( not (.a = .(x - y))),(.a = .(x - y)) IS NULL]) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) -- !case_when_two_side_6 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject[a IS NULL AS `a IS NULL`, coalesce(a, b) AS `coalesce(a, b)`, t1.a] +----PhysicalProject[coalesce(t1.a, t1.b) AS `coalesce(a, b)`, t1.a, t1.a IS NULL AS `a IS NULL`] ------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) @@ -208,28 +208,28 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----PhysicalResultSink ------PhysicalUnion --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.x = .coalesce(a, b))) otherCondition=((if(a IS NULL, x, y) = .coalesce(a, b))) +----------hashJoin[INNER_JOIN] hashCondition=((.x = .coalesce(a, b))) otherCondition=((if(.a IS NULL, .x, .y) = .coalesce(a, b))) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.y = .coalesce(a, b))) otherCondition=((if(a IS NULL, x, y) = .coalesce(a, b)) and OR[( not (x = coalesce(a, b))),(x = coalesce(a, b)) IS NULL]) +----------hashJoin[INNER_JOIN] hashCondition=((.y = .coalesce(a, b))) otherCondition=((if(.a IS NULL, .x, .y) = .coalesce(a, b)) and OR[( not (.x = .coalesce(a, b))),(.x = .coalesce(a, b)) IS NULL]) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) -- !case_when_two_side_7 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN](CASE WHEN (x > 10) THEN a WHEN (y > 10) THEN (a + b) ELSE 100 END = ((x + y) + -4)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a] +----NestedLoopJoin[INNER_JOIN](CASE WHEN (x > 10) THEN t1.a WHEN (y > 10) THEN (a + b) ELSE 100 END = ((x + y) + -4)) +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[((x + y) + -4) AS `((x + y) + -4)`, (x > 10) AS `(x > 10)`, (y > 10) AS `(y > 10)`, t2.x] +------PhysicalProject[((t2.x + t2.y) + -4) AS `((x + y) + -4)`, (t2.x > 10) AS `(x > 10)`, (t2.y > 10) AS `(y > 10)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !if_one_side_1 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN](if((a > x), a, b) = (a + b)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a, t1.b] +----NestedLoopJoin[INNER_JOIN](if((t1.a > t2.x), t1.a, t1.b) = (a + b)) +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a, t1.b] --------filter(OR[(t1.a = (t1.a + t1.b)),(t1.b = (t1.a + t1.b))]) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] ------PhysicalProject[t2.x] @@ -241,24 +241,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) -------PhysicalProject[(x + y) AS `(x + y)`, t2.x] +------PhysicalProject[(t2.x + t2.y) AS `(x + y)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] ----PhysicalResultSink ------PhysicalUnion --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.a = .(x + y))) otherCondition=((if((a > x), a, b) = .(x + y))) +----------hashJoin[INNER_JOIN] hashCondition=((.a = .(x + y))) otherCondition=((if((.a > .x), .a, .b) = .(x + y))) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) --------PhysicalProject[.a, .x] -----------hashJoin[INNER_JOIN] hashCondition=((.b = .(x + y))) otherCondition=((if((a > x), a, b) = .(x + y)) and OR[( not (a = (x + y))),(a = (x + y)) IS NULL]) +----------hashJoin[INNER_JOIN] hashCondition=((.b = .(x + y))) otherCondition=((if((.a > .x), .a, .b) = .(x + y)) and OR[( not (.a = .(x + y))),(.a = .(x + y)) IS NULL]) ------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------PhysicalCteConsumer ( cteId=CTEId#1 ) -- !ifnull_one_side_1 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN](ifnull(a, x) = (a + b)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a] +----NestedLoopJoin[INNER_JOIN](ifnull(t1.a, t2.x) = (a + b)) +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] ------PhysicalProject[t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] @@ -266,17 +266,17 @@ PhysicalResultSink -- !ifnull_two_side_1 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN](ifnull(a, x) = (x + y)) +----NestedLoopJoin[INNER_JOIN](ifnull(t1.a, t2.x) = (x + y)) ------PhysicalProject[t1.a] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x + y) AS `(x + y)`, t2.x] +------PhysicalProject[(t2.x + t2.y) AS `(x + y)`, t2.x] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] -- !nullif_one_side_1 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----NestedLoopJoin[INNER_JOIN](nullif(a, x) = (a + b)) -------PhysicalProject[(a + b) AS `(a + b)`, t1.a] +----NestedLoopJoin[INNER_JOIN](nullif(t1.a, t2.x) = (a + b)) +------PhysicalProject[(t1.a + t1.b) AS `(a + b)`, t1.a] --------filter((t1.a = (t1.a + t1.b))) ----------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] ------PhysicalProject[t2.x] @@ -285,9 +285,9 @@ PhysicalResultSink -- !nullif_two_side_1 -- PhysicalResultSink --PhysicalProject[t1.a, t2.x] -----hashJoin[INNER_JOIN] hashCondition=((t1.a = expr_(x + y))) otherCondition=((nullif(a, x) = (t2.x + t2.y))) +----hashJoin[INNER_JOIN] hashCondition=((t1.a = expr_(x + y))) otherCondition=((nullif(t1.a, t2.x) = (t2.x + t2.y))) ------PhysicalProject[t1.a] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_1(t1)] -------PhysicalProject[(x + y) AS `expr_(x + y)`, tbl_join_extract_or_from_case_when_2.x, tbl_join_extract_or_from_case_when_2.y] +------PhysicalProject[(t2.x + t2.y) AS `expr_(x + y)`, tbl_join_extract_or_from_case_when_2.x, tbl_join_extract_or_from_case_when_2.y] --------PhysicalOlapScan[tbl_join_extract_or_from_case_when_2(t2)] diff --git a/regression-test/data/nereids_rules_p0/limit_push_down/limit_push_down.out b/regression-test/data/nereids_rules_p0/limit_push_down/limit_push_down.out index 2b3d526546efbf..a8d29f8dff3aff 100644 --- a/regression-test/data/nereids_rules_p0/limit_push_down/limit_push_down.out +++ b/regression-test/data/nereids_rules_p0/limit_push_down/limit_push_down.out @@ -270,7 +270,7 @@ PhysicalResultSink ----PhysicalLimit[LOCAL] ------PhysicalUnion --------PhysicalLimit[LOCAL] -----------filter((cast(msg as DECIMALV3(38, 6)) > 100.000000)) +----------filter((cast(t1.msg as DECIMALV3(38, 6)) > 100.000000)) ------------PhysicalOlapScan[t1] --------PhysicalLimit[LOCAL] ----------filter((t2.id > 100)) diff --git a/regression-test/data/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.out b/regression-test/data/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.out index 7055064bf13bed..84ee70dd8b4d07 100644 --- a/regression-test/data/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.out +++ b/regression-test/data/nereids_rules_p0/limit_push_down/merge_topn_prefix_key.out @@ -9,3 +9,4 @@ PhysicalResultSink -- !merge_child_more_order_keys -- 12 1 2 13 1 3 + diff --git a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out index d46cbf4eacff94..170b8cae8bc0f4 100644 --- a/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out +++ b/regression-test/data/nereids_rules_p0/max_min_filter_push_down/max_min_filter_push_down.out @@ -30,7 +30,7 @@ PhysicalResultSink PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] -------filter((abs(value1) > 39)) +------filter((abs(max_min_filter_push_down1.value1) > 39)) --------PhysicalOlapScan[max_min_filter_push_down1] -- !min_commute -- diff --git a/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out b/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out index 6351649fbecf10..60aad1fd798ac6 100644 --- a/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out +++ b/regression-test/data/nereids_rules_p0/predicate_infer/infer_predicate.out @@ -354,19 +354,19 @@ PhysicalResultSink PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t12.id = t34.id)) otherCondition=() ----hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() -------filter(( not id IN (3, 4)) and (t12.id < 9) and (t12.id > 1)) +------filter(( not t12.id IN (3, 4)) and (t12.id < 9) and (t12.id > 1)) --------PhysicalOlapScan[t1] -------filter(( not id IN (3, 4)) and (t2.id < 9) and (t2.id > 1)) +------filter(( not t2.id IN (3, 4)) and (t2.id < 9) and (t2.id > 1)) --------PhysicalOlapScan[t2] ----hashJoin[INNER_JOIN] hashCondition=((t3.id = t4.id)) otherCondition=() -------filter(( not id IN (3, 4)) and (t3.id < 9) and (t3.id > 1)) +------filter(( not t3.id IN (3, 4)) and (t3.id < 9) and (t3.id > 1)) --------PhysicalOlapScan[t3] -------filter(( not id IN (3, 4)) and (t4.id < 9) and (t4.id > 1)) +------filter(( not t4.id IN (3, 4)) and (t4.id < 9) and (t4.id > 1)) --------PhysicalOlapScan[t4] -- !infer8 -- PhysicalResultSink ---NestedLoopJoin[INNER_JOIN]( not (id = id)) +--NestedLoopJoin[INNER_JOIN]( not (t1.id = t2.id)) ----filter((t1.id = 1)) ------PhysicalOlapScan[t1] ----PhysicalOlapScan[t2] @@ -384,7 +384,7 @@ PhysicalResultSink --NestedLoopJoin[CROSS_JOIN] ----filter((t1.id = 12345)) ------PhysicalOlapScan[t1] -----filter((cast(id as SMALLINT) = 12345)) +----filter((cast(t2.id as SMALLINT) = 12345)) ------PhysicalOlapScan[t2] -- !infer11 -- diff --git a/regression-test/data/nereids_rules_p0/pull_up_join_from_union/pull_up_join_from_union.out b/regression-test/data/nereids_rules_p0/pull_up_join_from_union/pull_up_join_from_union.out index 75c992d9bd9e7c..428bd276fc9bda 100644 --- a/regression-test/data/nereids_rules_p0/pull_up_join_from_union/pull_up_join_from_union.out +++ b/regression-test/data/nereids_rules_p0/pull_up_join_from_union/pull_up_join_from_union.out @@ -113,7 +113,7 @@ PhysicalResultSink ----PhysicalUnion ------PhysicalOlapScan[table_b(b)] ------PhysicalOlapScan[table_c(c)] -----filter((cast(value as DECIMALV3(38, 6)) = 1.000000)) +----filter((cast(a.value as DECIMALV3(38, 6)) = 1.000000)) ------PhysicalOlapScan[table_a(a)] -- !basic_join_union_res -- diff --git a/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out b/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out index a3726bd6d97034..f325fb50853b8c 100644 --- a/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out +++ b/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out @@ -2,61 +2,61 @@ -- !left_semi -- PhysicalResultSink --hashJoin[LEFT_SEMI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]]) -----filter(a IN (1, 2)) +----filter(t1.a IN (1, 2)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t1(t1)] -----filter(a IN (8, 9)) +----filter(t2.a IN (8, 9)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t2(t2)] -- !right_semi -- PhysicalResultSink --hashJoin[RIGHT_SEMI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]]) -----filter(a IN (1, 2)) +----filter(t1.a IN (1, 2)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t1(t1)] -----filter(a IN (8, 9)) +----filter(t2.a IN (8, 9)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t2(t2)] -- !left -- PhysicalResultSink ---hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (1, 2)) +--hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and t1.a IN (1, 2)) ----PhysicalOlapScan[extract_from_disjunction_in_join_t1(t1)] -----filter(a IN (8, 9)) +----filter(t2.a IN (8, 9)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t2(t2)] -- !right -- PhysicalResultSink ---hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (8, 9)) -----filter(a IN (1, 2)) +--hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and t2.a IN (8, 9)) +----filter(t1.a IN (1, 2)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t1(t1)] ----PhysicalOlapScan[extract_from_disjunction_in_join_t2(t2)] -- !left_anti -- PhysicalResultSink ---hashJoin[LEFT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (1, 2)) +--hashJoin[LEFT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and t1.a IN (1, 2)) ----PhysicalOlapScan[extract_from_disjunction_in_join_t1(t1)] -----filter(a IN (8, 9)) +----filter(t2.a IN (8, 9)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t2(t2)] -- !right_anti -- PhysicalResultSink ---hashJoin[RIGHT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (8, 9)) -----filter(a IN (1, 2)) +--hashJoin[RIGHT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and t2.a IN (8, 9)) +----filter(t1.a IN (1, 2)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t1(t1)] ----PhysicalOlapScan[extract_from_disjunction_in_join_t2(t2)] -- !inner -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]]) -----filter(a IN (1, 2)) +----filter(t1.a IN (1, 2)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t1(t1)] -----filter(a IN (8, 9)) +----filter(t2.a IN (8, 9)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t2(t2)] -- !outer -- PhysicalResultSink ---hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and a IN (1, 2)) +--hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=(OR[AND[(t2.a = 9),(t1.a = 1)],AND[(t1.a = 2),(t2.a = 8)]] and t1.a IN (1, 2)) ----filter((t1.c = 3)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t1(t1)] -----filter(a IN (8, 9)) +----filter(t2.a IN (8, 9)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t2(t2)] -- !left_semi_res -- diff --git a/regression-test/data/nereids_rules_p0/push_down_filter/push_down_filter_through_window.out b/regression-test/data/nereids_rules_p0/push_down_filter/push_down_filter_through_window.out index 7cddd539ff1c5d..61ad8adedcae53 100644 --- a/regression-test/data/nereids_rules_p0/push_down_filter/push_down_filter_through_window.out +++ b/regression-test/data/nereids_rules_p0/push_down_filter/push_down_filter_through_window.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalQuickSort[LOCAL_SORT] --------PhysicalPartitionTopN ----------PhysicalProject -------------filter((abs((cast(id as BIGINT) + cast(value1 as BIGINT))) < 4)) +------------filter((abs((cast(t.id as BIGINT) + cast(t.value1 as BIGINT))) < 4)) --------------PhysicalOlapScan[push_down_multi_column_predicate_through_window_t] -- !multi_column_predicate_push_down_window -- diff --git a/regression-test/data/nereids_rules_p0/sumRewrite.out b/regression-test/data/nereids_rules_p0/sumRewrite.out index 265ccae7deab75..e4f7b832c4140b 100644 --- a/regression-test/data/nereids_rules_p0/sumRewrite.out +++ b/regression-test/data/nereids_rules_p0/sumRewrite.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sum_add_const_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) + 2)) AS `sum(id + 2)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.id as BIGINT) + 2)) AS `sum(id + 2)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`)] ------PhysicalProject[sr.id] --------PhysicalOlapScan[sr] @@ -11,8 +11,8 @@ PhysicalResultSink -- !sum_add_const_alias_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) + 2)) AS `result`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.id as BIGINT) + 2)) AS `result`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`)] ------PhysicalProject[sr.id] --------PhysicalOlapScan[sr] @@ -21,10 +21,10 @@ PhysicalResultSink -- !sum_add_const_where_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) + 2)) AS `sum(id + 2)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.id as BIGINT) + 2)) AS `sum(id + 2)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`)] ------PhysicalProject[sr.id] ---------filter(( not id IS NULL)) +--------filter(( not sr.id IS NULL)) ----------PhysicalOlapScan[sr] -- !sum_add_const_where_result -- @@ -32,8 +32,8 @@ PhysicalResultSink -- !sum_add_const_group_by_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(id as BIGINT) + 2)) AS `sum(id + 2)`)] -----hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`, sr.not_null_id)] +--hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(sr.id as BIGINT) + 2)) AS `sum(id + 2)`)] +----hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(sr.id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`, sr.not_null_id)] ------PhysicalProject[sr.id, sr.not_null_id] --------PhysicalOlapScan[sr] @@ -47,8 +47,8 @@ PhysicalResultSink -- !sum_add_const_having_shape -- PhysicalResultSink --filter((sum(id + 2) > 5)) -----hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(id as BIGINT) + 2)) AS `sum(id + 2)`)] -------hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`, sr.not_null_id)] +----hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(sr.id as BIGINT) + 2)) AS `sum(id + 2)`)] +------hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(sr.id as BIGINT) + 2)) AS `partial_sum((cast(id as BIGINT) + 2))`, sr.not_null_id)] --------PhysicalProject[sr.id, sr.not_null_id] ----------PhysicalOlapScan[sr] @@ -61,8 +61,8 @@ PhysicalResultSink -- !sum_sub_const_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) - 2)) AS `sum(id - 2)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.id as BIGINT) - 2)) AS `sum(id - 2)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`)] ------PhysicalProject[sr.id] --------PhysicalOlapScan[sr] @@ -71,8 +71,8 @@ PhysicalResultSink -- !sum_sub_const_alias_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) - 2)) AS `result`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.id as BIGINT) - 2)) AS `result`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`)] ------PhysicalProject[sr.id] --------PhysicalOlapScan[sr] @@ -81,10 +81,10 @@ PhysicalResultSink -- !sum_sub_const_where_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) - 2)) AS `sum(id - 2)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.id as BIGINT) - 2)) AS `sum(id - 2)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`)] ------PhysicalProject[sr.id] ---------filter(( not id IS NULL)) +--------filter(( not sr.id IS NULL)) ----------PhysicalOlapScan[sr] -- !sum_sub_const_where_result -- @@ -92,8 +92,8 @@ PhysicalResultSink -- !sum_sub_const_group_by_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(id as BIGINT) - 2)) AS `sum(id - 2)`)] -----hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`, sr.not_null_id)] +--hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(sr.id as BIGINT) - 2)) AS `sum(id - 2)`)] +----hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(sr.id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`, sr.not_null_id)] ------PhysicalProject[sr.id, sr.not_null_id] --------PhysicalOlapScan[sr] @@ -107,8 +107,8 @@ PhysicalResultSink -- !sum_sub_const_having_shape -- PhysicalResultSink --filter((sum(id - 2) > 0)) -----hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(id as BIGINT) - 2)) AS `sum(id - 2)`)] -------hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`, sr.not_null_id)] +----hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(sr.id as BIGINT) - 2)) AS `sum(id - 2)`)] +------hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(sr.id as BIGINT) - 2)) AS `partial_sum((cast(id as BIGINT) - 2))`, sr.not_null_id)] --------PhysicalProject[sr.id, sr.not_null_id] ----------PhysicalOlapScan[sr] @@ -121,7 +121,7 @@ PhysicalResultSink -- !sum_add_const_empty_table_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) + 2)) AS `sum(id + 2)`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.id as BIGINT) + 2)) AS `sum(id + 2)`)] ----PhysicalEmptyRelation -- !sum_add_const_empty_table_result -- @@ -135,7 +135,7 @@ PhysicalResultSink -- !sum_sub_const_empty_table_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) - 2)) AS `sum(id - 2)`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.id as BIGINT) - 2)) AS `sum(id - 2)`)] ----PhysicalEmptyRelation -- !sum_sub_const_empty_table_result -- @@ -149,8 +149,8 @@ PhysicalResultSink -- !float_sum_add_const_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(f_id as DOUBLE) + 2.0)) AS `sum(f_id + 2)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `sum(f_id + 2)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`)] ------PhysicalProject[sr.f_id] --------PhysicalOlapScan[sr] @@ -159,8 +159,8 @@ PhysicalResultSink -- !float_sum_add_const_alias_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(f_id as DOUBLE) + 2.0)) AS `result`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `result`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`)] ------PhysicalProject[sr.f_id] --------PhysicalOlapScan[sr] @@ -169,10 +169,10 @@ PhysicalResultSink -- !float_sum_add_const_where_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(f_id as DOUBLE) + 2.0)) AS `sum(f_id + 2)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `sum(f_id + 2)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`)] ------PhysicalProject[sr.f_id] ---------filter(( not f_id IS NULL)) +--------filter(( not sr.f_id IS NULL)) ----------PhysicalOlapScan[sr] -- !float_sum_add_const_where_result -- @@ -180,8 +180,8 @@ PhysicalResultSink -- !float_sum_add_const_group_by_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(f_id as DOUBLE) + 2.0)) AS `sum(f_id + 2)`)] -----hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`, sr.not_null_id)] +--hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `sum(f_id + 2)`)] +----hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`, sr.not_null_id)] ------PhysicalProject[sr.f_id, sr.not_null_id] --------PhysicalOlapScan[sr] @@ -195,8 +195,8 @@ PhysicalResultSink -- !float_sum_add_const_having_shape -- PhysicalResultSink --filter((sum(f_id + 2) > 5.0)) -----hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(f_id as DOUBLE) + 2.0)) AS `sum(f_id + 2)`)] -------hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`, sr.not_null_id)] +----hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `sum(f_id + 2)`)] +------hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) + 2.0)) AS `partial_sum((cast(f_id as DOUBLE) + 2.0))`, sr.not_null_id)] --------PhysicalProject[sr.f_id, sr.not_null_id] ----------PhysicalOlapScan[sr] @@ -209,8 +209,8 @@ PhysicalResultSink -- !float_sum_sub_const_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(f_id as DOUBLE) - 2.0)) AS `sum(f_id - 2)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `sum(f_id - 2)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`)] ------PhysicalProject[sr.f_id] --------PhysicalOlapScan[sr] @@ -219,8 +219,8 @@ PhysicalResultSink -- !float_sum_sub_const_alias_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(f_id as DOUBLE) - 2.0)) AS `result`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `result`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`)] ------PhysicalProject[sr.f_id] --------PhysicalOlapScan[sr] @@ -229,10 +229,10 @@ PhysicalResultSink -- !float_sum_sub_const_where_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(f_id as DOUBLE) - 2.0)) AS `sum(f_id - 2)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `sum(f_id - 2)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`)] ------PhysicalProject[sr.f_id] ---------filter(( not f_id IS NULL)) +--------filter(( not sr.f_id IS NULL)) ----------PhysicalOlapScan[sr] -- !float_sum_sub_const_where_result -- @@ -240,8 +240,8 @@ PhysicalResultSink -- !float_sum_sub_const_group_by_shape -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(f_id as DOUBLE) - 2.0)) AS `sum(f_id - 2)`)] -----hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`, sr.not_null_id)] +--hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `sum(f_id - 2)`)] +----hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`, sr.not_null_id)] ------PhysicalProject[sr.f_id, sr.not_null_id] --------PhysicalOlapScan[sr] @@ -255,8 +255,8 @@ PhysicalResultSink -- !float_sum_sub_const_having_shape -- PhysicalResultSink --filter((sum(f_id - 2) > 0.0)) -----hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(f_id as DOUBLE) - 2.0)) AS `sum(f_id - 2)`)] -------hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`, sr.not_null_id)] +----hashAgg[GLOBAL, groupByExpr=(sr.not_null_id), outputExpr=(sr.not_null_id, sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `sum(f_id - 2)`)] +------hashAgg[LOCAL, groupByExpr=(sr.not_null_id), outputExpr=(partial_sum((cast(sr.f_id as DOUBLE) - 2.0)) AS `partial_sum((cast(f_id as DOUBLE) - 2.0))`, sr.not_null_id)] --------PhysicalProject[sr.f_id, sr.not_null_id] ----------PhysicalOlapScan[sr] @@ -269,8 +269,8 @@ PhysicalResultSink -- !sum_null_and_not_null_shape -- PhysicalResultSink --PhysicalProject[(sum(id) + (count(id) * 1)) AS `sum(id + 1)`, (sum(id) - (count(id) * 1)) AS `sum(id - 1)`, (sum(not_null_id) + (count(not_null_id) * 1)) AS `sum(not_null_id + 1)`, (sum(not_null_id) - (count(not_null_id) * 1)) AS `sum(not_null_id - 1)`, sum(id), sum(not_null_id)] -----hashAgg[GLOBAL, groupByExpr=(), outputExpr=(count(id) AS `count(id)`, count(not_null_id) AS `count(not_null_id)`, sum(id) AS `sum(id)`, sum(not_null_id) AS `sum(not_null_id)`)] -------hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_count(id) AS `partial_count(id)`, partial_count(not_null_id) AS `partial_count(not_null_id)`, partial_sum(id) AS `partial_sum(id)`, partial_sum(not_null_id) AS `partial_sum(not_null_id)`)] +----hashAgg[GLOBAL, groupByExpr=(), outputExpr=(count(sr.id) AS `count(id)`, count(sr.not_null_id) AS `count(not_null_id)`, sum(sr.id) AS `sum(id)`, sum(sr.not_null_id) AS `sum(not_null_id)`)] +------hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_count(sr.id) AS `partial_count(id)`, partial_count(sr.not_null_id) AS `partial_count(not_null_id)`, partial_sum(sr.id) AS `partial_sum(id)`, partial_sum(sr.not_null_id) AS `partial_sum(not_null_id)`)] --------PhysicalProject[sr.id, sr.not_null_id] ----------PhysicalOlapScan[sr] @@ -280,26 +280,26 @@ PhysicalResultSink -- !sum_distinct_shape -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject[(cast(id as BIGINT) + 1) AS `(cast(id as BIGINT) + 1)`, sr.id] +----PhysicalProject[(cast(sr.id as BIGINT) + 1) AS `(cast(id as BIGINT) + 1)`, sr.id] ------PhysicalOlapScan[sr] --PhysicalResultSink ----NestedLoopJoin[CROSS_JOIN] ------NestedLoopJoin[CROSS_JOIN] --------PhysicalProject[sum(DISTINCT id) AS `sum(distinct id)`] -----------hashAgg[DISTINCT_GLOBAL, groupByExpr=(), outputExpr=(sum(id) AS `sum(DISTINCT id)`)] -------------hashAgg[DISTINCT_LOCAL, groupByExpr=(), outputExpr=(partial_sum(id) AS `partial_sum(id)`)] +----------hashAgg[DISTINCT_GLOBAL, groupByExpr=(), outputExpr=(sum(.id) AS `sum(DISTINCT id)`)] +------------hashAgg[DISTINCT_LOCAL, groupByExpr=(), outputExpr=(partial_sum(.id) AS `partial_sum(id)`)] --------------hashAgg[GLOBAL, groupByExpr=(.id), outputExpr=(.id)] ----------------hashAgg[LOCAL, groupByExpr=(.id), outputExpr=(.id)] ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------PhysicalProject[sum(DISTINCT (cast(id as BIGINT) + 1)) AS `sum(distinct id + 1)`] -----------hashAgg[DISTINCT_GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) + 1)) AS `sum(DISTINCT (cast(id as BIGINT) + 1))`)] -------------hashAgg[DISTINCT_LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(id as BIGINT) + 1)) AS `partial_sum((cast(id as BIGINT) + 1))`)] +----------hashAgg[DISTINCT_GLOBAL, groupByExpr=(), outputExpr=(sum(.(cast(id as BIGINT) + 1)) AS `sum(DISTINCT (cast(id as BIGINT) + 1))`)] +------------hashAgg[DISTINCT_LOCAL, groupByExpr=(), outputExpr=(partial_sum(.(cast(id as BIGINT) + 1)) AS `partial_sum((cast(id as BIGINT) + 1))`)] --------------hashAgg[GLOBAL, groupByExpr=(.(cast(id as BIGINT) + 1)), outputExpr=(.(cast(id as BIGINT) + 1))] ----------------hashAgg[LOCAL, groupByExpr=(.(cast(id as BIGINT) + 1)), outputExpr=(.(cast(id as BIGINT) + 1))] ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------PhysicalProject[sum((cast(id as BIGINT) + 1)) AS `sum(id + 1)`, sum(id) AS `sum(id)`] ---------hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(id as BIGINT) + 1)) AS `sum((cast(id as BIGINT) + 1))`, sum(id) AS `sum(id)`)] -----------hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(id as BIGINT) + 1)) AS `partial_sum((cast(id as BIGINT) + 1))`, partial_sum(id) AS `partial_sum(id)`)] +--------hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum(.(cast(id as BIGINT) + 1)) AS `sum((cast(id as BIGINT) + 1))`, sum(.id) AS `sum(id)`)] +----------hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum(.(cast(id as BIGINT) + 1)) AS `partial_sum((cast(id as BIGINT) + 1))`, partial_sum(.id) AS `partial_sum(id)`)] ------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !sum_distinct_result -- @@ -308,26 +308,26 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) -- !sum_not_null_distinct_shape -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject[(cast(not_null_id as BIGINT) + 1) AS `(cast(not_null_id as BIGINT) + 1)`, sr.not_null_id] +----PhysicalProject[(cast(sr.not_null_id as BIGINT) + 1) AS `(cast(not_null_id as BIGINT) + 1)`, sr.not_null_id] ------PhysicalOlapScan[sr] --PhysicalResultSink ----NestedLoopJoin[CROSS_JOIN] ------NestedLoopJoin[CROSS_JOIN] --------PhysicalProject[sum(DISTINCT not_null_id) AS `sum(distinct not_null_id)`] -----------hashAgg[DISTINCT_GLOBAL, groupByExpr=(), outputExpr=(sum(not_null_id) AS `sum(DISTINCT not_null_id)`)] -------------hashAgg[DISTINCT_LOCAL, groupByExpr=(), outputExpr=(partial_sum(not_null_id) AS `partial_sum(not_null_id)`)] +----------hashAgg[DISTINCT_GLOBAL, groupByExpr=(), outputExpr=(sum(.not_null_id) AS `sum(DISTINCT not_null_id)`)] +------------hashAgg[DISTINCT_LOCAL, groupByExpr=(), outputExpr=(partial_sum(.not_null_id) AS `partial_sum(not_null_id)`)] --------------hashAgg[GLOBAL, groupByExpr=(.not_null_id), outputExpr=(.not_null_id)] ----------------hashAgg[LOCAL, groupByExpr=(.not_null_id), outputExpr=(.not_null_id)] ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------PhysicalProject[sum(DISTINCT (cast(not_null_id as BIGINT) + 1)) AS `sum(distinct not_null_id + 1)`] -----------hashAgg[DISTINCT_GLOBAL, groupByExpr=(), outputExpr=(sum((cast(not_null_id as BIGINT) + 1)) AS `sum(DISTINCT (cast(not_null_id as BIGINT) + 1))`)] -------------hashAgg[DISTINCT_LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(not_null_id as BIGINT) + 1)) AS `partial_sum((cast(not_null_id as BIGINT) + 1))`)] +----------hashAgg[DISTINCT_GLOBAL, groupByExpr=(), outputExpr=(sum(.(cast(not_null_id as BIGINT) + 1)) AS `sum(DISTINCT (cast(not_null_id as BIGINT) + 1))`)] +------------hashAgg[DISTINCT_LOCAL, groupByExpr=(), outputExpr=(partial_sum(.(cast(not_null_id as BIGINT) + 1)) AS `partial_sum((cast(not_null_id as BIGINT) + 1))`)] --------------hashAgg[GLOBAL, groupByExpr=(.(cast(not_null_id as BIGINT) + 1)), outputExpr=(.(cast(not_null_id as BIGINT) + 1))] ----------------hashAgg[LOCAL, groupByExpr=(.(cast(not_null_id as BIGINT) + 1)), outputExpr=(.(cast(not_null_id as BIGINT) + 1))] ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------PhysicalProject[sum((cast(not_null_id as BIGINT) + 1)) AS `sum(not_null_id + 1)`, sum(not_null_id) AS `sum(not_null_id)`] ---------hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum((cast(not_null_id as BIGINT) + 1)) AS `sum((cast(not_null_id as BIGINT) + 1))`, sum(not_null_id) AS `sum(not_null_id)`)] -----------hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum((cast(not_null_id as BIGINT) + 1)) AS `partial_sum((cast(not_null_id as BIGINT) + 1))`, partial_sum(not_null_id) AS `partial_sum(not_null_id)`)] +--------hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum(.(cast(not_null_id as BIGINT) + 1)) AS `sum((cast(not_null_id as BIGINT) + 1))`, sum(.not_null_id) AS `sum(not_null_id)`)] +----------hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum(.(cast(not_null_id as BIGINT) + 1)) AS `partial_sum((cast(not_null_id as BIGINT) + 1))`, partial_sum(.not_null_id) AS `partial_sum(not_null_id)`)] ------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !sum_not_null_distinct_result -- diff --git a/regression-test/data/nereids_rules_p0/unique_function/add_project_for_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/add_project_for_unique_function.out index 1416e712229536..b2f0a6661447a5 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/add_project_for_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/add_project_for_unique_function.out @@ -15,25 +15,25 @@ PhysicalResultSink -- !project_1 -- PhysicalResultSink ---PhysicalProject[((cast(id as BIGINT) + random(1, 100)) > 20) AS `id + random(1, 100) > 20`, (cast(id as BIGINT) * 200) AS `id * 200`] +--PhysicalProject[((cast(t1.id as BIGINT) + random(1, 100)) > 20) AS `id + random(1, 100) > 20`, (cast(t1.id as BIGINT) * 200) AS `id * 200`] ----PhysicalOlapScan[t1] -- !project_2 -- PhysicalResultSink ---PhysicalProject[(cast(id as BIGINT) * 200) AS `id * 200`, AND[((cast(id as BIGINT) + $_random_5_$) >= 10),((cast(id as BIGINT) + $_random_5_$) <= 20)] AS `id + random(1, 100) between 10 and 20`] +--PhysicalProject[(cast(t1.id as BIGINT) * 200) AS `id * 200`, AND[((cast(t1.id as BIGINT) + $_random_5_$) >= 10),((cast(t1.id as BIGINT) + $_random_5_$) <= 20)] AS `id + random(1, 100) between 10 and 20`] ----PhysicalProject[random(1, 100) AS `$_random_5_$`, t1.id] ------PhysicalOlapScan[t1] -- !filter_1 -- PhysicalResultSink --PhysicalProject[t1.id] -----filter(((cast(id as BIGINT) + random(1, 100)) >= 10)) +----filter(((cast(t1.id as BIGINT) + random(1, 100)) >= 10)) ------PhysicalOlapScan[t1] -- !filter_2 -- PhysicalResultSink --PhysicalProject[t1.id] -----filter(((cast(id as BIGINT) + $_random_3_$) <= 20) and ((cast(id as BIGINT) + $_random_3_$) >= 10)) +----filter(((cast(t1.id as BIGINT) + $_random_3_$) <= 20) and ((cast(t1.id as BIGINT) + $_random_3_$) >= 10)) ------PhysicalProject[random(1, 100) AS `$_random_3_$`, t1.id] --------PhysicalOlapScan[t1] @@ -50,7 +50,7 @@ PhysicalResultSink --hashAgg[GLOBAL, groupByExpr=(k), outputExpr=(k)] ----hashAgg[LOCAL, groupByExpr=(k), outputExpr=(k)] ------PhysicalUnion(constantExprsList=[[TRUE AS `true`]]) ---------PhysicalProject[AND[((cast(id as DOUBLE) + $_random_9_$) >= 0.1),((cast(id as DOUBLE) + $_random_9_$) <= 0.5)] AS `k`] +--------PhysicalProject[AND[((cast(t1.id as DOUBLE) + $_random_9_$) >= 0.1),((cast(t1.id as DOUBLE) + $_random_9_$) <= 0.5)] AS `k`] ----------PhysicalProject[random() AS `$_random_9_$`, t1.id] ------------PhysicalOlapScan[t1] @@ -63,7 +63,7 @@ PhysicalResultSink -- !union_all_2 -- PhysicalResultSink --PhysicalUnion(constantExprsList=[[TRUE AS `true`]]) -----PhysicalProject[AND[((cast(id as DOUBLE) + $_random_7_$) >= 0.1),((cast(id as DOUBLE) + $_random_7_$) <= 0.5)] AS `k`] +----PhysicalProject[AND[((cast(t1.id as DOUBLE) + $_random_7_$) >= 0.1),((cast(t1.id as DOUBLE) + $_random_7_$) <= 0.5)] AS `k`] ------PhysicalProject[random() AS `$_random_7_$`, t1.id] --------PhysicalOlapScan[t1] @@ -78,8 +78,8 @@ PhysicalResultSink -- !intersect_2 -- PhysicalResultSink --PhysicalIntersect -----PhysicalProject[AND[((cast(id as DOUBLE) + $_random_7_$) >= 0.1),((cast(id as DOUBLE) + $_random_7_$) <= 0.5)] AS `k`] -------filter(((cast(id as DOUBLE) + $_random_7_$) <= 0.5) and ((cast(id as DOUBLE) + $_random_7_$) >= 0.1)) +----PhysicalProject[AND[((cast(t1.id as DOUBLE) + $_random_7_$) >= 0.1),((cast(t1.id as DOUBLE) + $_random_7_$) <= 0.5)] AS `k`] +------filter(((cast(t1.id as DOUBLE) + $_random_7_$) <= 0.5) and ((cast(t1.id as DOUBLE) + $_random_7_$) >= 0.1)) --------PhysicalProject[random() AS `$_random_7_$`, t1.id] ----------PhysicalOlapScan[t1] ----PhysicalOneRowRelation[TRUE AS `true`] @@ -94,7 +94,7 @@ PhysicalResultSink -- !except_2 -- PhysicalResultSink --PhysicalExcept -----PhysicalProject[AND[((cast(id as DOUBLE) + $_random_7_$) >= 0.1),((cast(id as DOUBLE) + $_random_7_$) <= 0.5)] AS `k`] +----PhysicalProject[AND[((cast(t1.id as DOUBLE) + $_random_7_$) >= 0.1),((cast(t1.id as DOUBLE) + $_random_7_$) <= 0.5)] AS `k`] ------PhysicalProject[random() AS `$_random_7_$`, t1.id] --------PhysicalOlapScan[t1] ----PhysicalOneRowRelation[TRUE AS `true`] @@ -104,7 +104,7 @@ PhysicalResultSink --PhysicalProject[t.k] ----PhysicalQuickSort[MERGE_SORT, orderKeys=(AND[((cast(k as DOUBLE) + random(100)) >= 0.6),((cast(k as DOUBLE) + random(100)) <= 0.7)] asc null first)] ------PhysicalQuickSort[LOCAL_SORT, orderKeys=(AND[((cast(k as DOUBLE) + random(100)) >= 0.6),((cast(k as DOUBLE) + random(100)) <= 0.7)] asc null first)] ---------PhysicalProject[AND[((cast(k as DOUBLE) + $_random_5_$) >= 0.6),((cast(k as DOUBLE) + $_random_5_$) <= 0.7)] AS `AND[((cast(k as DOUBLE) + random(100)) >= 0.6),((cast(k as DOUBLE) + random(100)) <= 0.7)]`, t.k] +--------PhysicalProject[AND[((cast(t.k as DOUBLE) + $_random_5_$) >= 0.6),((cast(t.k as DOUBLE) + $_random_5_$) <= 0.7)] AS `AND[((cast(k as DOUBLE) + random(100)) >= 0.6),((cast(k as DOUBLE) + random(100)) <= 0.7)]`, t.k] ----------PhysicalProject[AND[($_random_6_$ >= 0.1),($_random_6_$ <= 0.5)] AS `k`, random(100) AS `$_random_5_$`] ------------PhysicalOneRowRelation[random() AS `$_random_6_$`] @@ -113,8 +113,8 @@ PhysicalResultSink --PhysicalProject[t.k] ----PhysicalQuickSort[MERGE_SORT, orderKeys=(AND[((cast(k as DOUBLE) + random(100)) >= 0.6),((cast(k as DOUBLE) + random(100)) <= 0.7)] asc null first)] ------PhysicalQuickSort[LOCAL_SORT, orderKeys=(AND[((cast(k as DOUBLE) + random(100)) >= 0.6),((cast(k as DOUBLE) + random(100)) <= 0.7)] asc null first)] ---------PhysicalProject[AND[((cast(k as DOUBLE) + $_random_6_$) >= 0.6),((cast(k as DOUBLE) + $_random_6_$) <= 0.7)] AS `AND[((cast(k as DOUBLE) + random(100)) >= 0.6),((cast(k as DOUBLE) + random(100)) <= 0.7)]`, t.k] -----------PhysicalProject[AND[((cast(id as DOUBLE) + $_random_7_$) >= 0.1),((cast(id as DOUBLE) + $_random_7_$) <= 0.5)] AS `k`, random(100) AS `$_random_6_$`] +--------PhysicalProject[AND[((cast(t.k as DOUBLE) + $_random_6_$) >= 0.6),((cast(t.k as DOUBLE) + $_random_6_$) <= 0.7)] AS `AND[((cast(k as DOUBLE) + random(100)) >= 0.6),((cast(k as DOUBLE) + random(100)) <= 0.7)]`, t.k] +----------PhysicalProject[AND[((cast(t1.id as DOUBLE) + $_random_7_$) >= 0.1),((cast(t1.id as DOUBLE) + $_random_7_$) <= 0.5)] AS `k`, random(100) AS `$_random_6_$`] ------------PhysicalProject[random() AS `$_random_7_$`, t1.id] --------------PhysicalOlapScan[t1] @@ -125,16 +125,16 @@ PhysicalResultSink -- !agg_2 -- PhysicalResultSink ---hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum(cast(AND[($_random_5_$ >= 0.6),($_random_5_$ <= 0.7)] as TINYINT)) AS `sum(random(100) between 0.6 and 0.7)`, sum(id) AS `sum(id)`)] -----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum(cast(AND[($_random_5_$ >= 0.6),($_random_5_$ <= 0.7)] as TINYINT)) AS `partial_sum(cast(AND[($_random_5_$ >= 0.6),($_random_5_$ <= 0.7)] as TINYINT))`, partial_sum(id) AS `partial_sum(id)`)] +--hashAgg[GLOBAL, groupByExpr=(), outputExpr=(sum(cast(AND[($_random_5_$ >= 0.6),($_random_5_$ <= 0.7)] as TINYINT)) AS `sum(random(100) between 0.6 and 0.7)`, sum(t1.id) AS `sum(id)`)] +----hashAgg[LOCAL, groupByExpr=(), outputExpr=(partial_sum(cast(AND[($_random_5_$ >= 0.6),($_random_5_$ <= 0.7)] as TINYINT)) AS `partial_sum(cast(AND[($_random_5_$ >= 0.6),($_random_5_$ <= 0.7)] as TINYINT))`, partial_sum(t1.id) AS `partial_sum(id)`)] ------PhysicalProject[random(100) AS `$_random_5_$`, t1.id] --------PhysicalOlapScan[t1] -- !agg_3 -- PhysicalResultSink --PhysicalProject[sum(id), sum(random(100) between 0.6 and 0.7)] -----hashAgg[GLOBAL, groupByExpr=(AND[(random() >= cast(0.1 as DOUBLE)),(random() <= cast(0.5 as DOUBLE))]), outputExpr=(AND[(random() >= cast(0.1 as DOUBLE)),(random() <= cast(0.5 as DOUBLE))], sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT)) AS `sum(random(100) between 0.6 and 0.7)`, sum(id) AS `sum(id)`)] -------hashAgg[LOCAL, groupByExpr=(AND[(random() >= cast(0.1 as DOUBLE)),(random() <= cast(0.5 as DOUBLE))]), outputExpr=(AND[(random() >= cast(0.1 as DOUBLE)),(random() <= cast(0.5 as DOUBLE))], partial_sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT)) AS `partial_sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT))`, partial_sum(id) AS `partial_sum(id)`)] +----hashAgg[GLOBAL, groupByExpr=(AND[(random() >= cast(0.1 as DOUBLE)),(random() <= cast(0.5 as DOUBLE))]), outputExpr=(AND[(random() >= cast(0.1 as DOUBLE)),(random() <= cast(0.5 as DOUBLE))], sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT)) AS `sum(random(100) between 0.6 and 0.7)`, sum(t1.id) AS `sum(id)`)] +------hashAgg[LOCAL, groupByExpr=(AND[(random() >= cast(0.1 as DOUBLE)),(random() <= cast(0.5 as DOUBLE))]), outputExpr=(AND[(random() >= cast(0.1 as DOUBLE)),(random() <= cast(0.5 as DOUBLE))], partial_sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT)) AS `partial_sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT))`, partial_sum(t1.id) AS `partial_sum(id)`)] --------PhysicalProject[AND[($_random_8_$ >= 0.1),($_random_8_$ <= 0.5)] AS `AND[(random() >= cast(0.1 as DOUBLE)),(random() <= cast(0.5 as DOUBLE))]`, random(100) AS `$_random_7_$`, t1.id] ----------PhysicalProject[random() AS `$_random_8_$`, t1.id] ------------PhysicalOlapScan[t1] @@ -142,9 +142,9 @@ PhysicalResultSink -- !agg_4 -- PhysicalResultSink --PhysicalProject[sum(id), sum(random(100) between 0.6 and 0.7)] -----hashAgg[GLOBAL, groupByExpr=(AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))]), outputExpr=(AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))], sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT)) AS `sum(random(100) between 0.6 and 0.7)`, sum(id) AS `sum(id)`)] -------hashAgg[LOCAL, groupByExpr=(AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))]), outputExpr=(AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))], partial_sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT)) AS `partial_sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT))`, partial_sum(id) AS `partial_sum(id)`)] ---------PhysicalProject[AND[((cast(id as DOUBLE) + $_random_8_$) >= 0.1),((cast(id as DOUBLE) + $_random_8_$) <= 0.5)] AS `AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))]`, random(100) AS `$_random_7_$`, t1.id] +----hashAgg[GLOBAL, groupByExpr=(AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))]), outputExpr=(AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))], sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT)) AS `sum(random(100) between 0.6 and 0.7)`, sum(t1.id) AS `sum(id)`)] +------hashAgg[LOCAL, groupByExpr=(AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))]), outputExpr=(AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))], partial_sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT)) AS `partial_sum(cast(AND[($_random_7_$ >= 0.6),($_random_7_$ <= 0.7)] as TINYINT))`, partial_sum(t1.id) AS `partial_sum(id)`)] +--------PhysicalProject[AND[((cast(t1.id as DOUBLE) + $_random_8_$) >= 0.1),((cast(t1.id as DOUBLE) + $_random_8_$) <= 0.5)] AS `AND[((cast(id as DOUBLE) + random()) >= cast(0.1 as DOUBLE)),((cast(id as DOUBLE) + random()) <= cast(0.5 as DOUBLE))]`, random(100) AS `$_random_7_$`, t1.id] ----------PhysicalProject[random() AS `$_random_8_$`, t1.id] ------------PhysicalOlapScan[t1] @@ -172,7 +172,7 @@ PhysicalResultSink over(partition by id + random(2) between 0.2 and 0.22 order by id + random(3) between 0.3 and 0.33)] ----PhysicalWindow ------PhysicalQuickSort[LOCAL_SORT, orderKeys=(AND[((cast(id as DOUBLE) + random(2)) >= 0.2),((cast(id as DOUBLE) + random(2)) <= 0.22)] asc, AND[((cast(id as DOUBLE) + random(3)) >= 0.3),((cast(id as DOUBLE) + random(3)) <= 0.33)] asc null first)] ---------PhysicalProject[AND[((cast(id as DOUBLE) + $_random_10_$) >= 0.2),((cast(id as DOUBLE) + $_random_10_$) <= 0.22)] AS `AND[((cast(id as DOUBLE) + random(2)) >= 0.2),((cast(id as DOUBLE) + random(2)) <= 0.22)]`, AND[((cast(id as DOUBLE) + $_random_11_$) >= 0.3),((cast(id as DOUBLE) + $_random_11_$) <= 0.33)] AS `AND[((cast(id as DOUBLE) + random(3)) >= 0.3),((cast(id as DOUBLE) + random(3)) <= 0.33)]`, cast(AND[((cast(id as DOUBLE) + $_random_9_$) >= 0.1),((cast(id as DOUBLE) + $_random_9_$) <= 0.11)] as TINYINT) AS `cast(AND[((cast(id as DOUBLE) + random(1)) >= 0.1),((cast(id as DOUBLE) + random(1)) <= 0.11)] as TINYINT)`] +--------PhysicalProject[AND[((cast(t1.id as DOUBLE) + $_random_10_$) >= 0.2),((cast(t1.id as DOUBLE) + $_random_10_$) <= 0.22)] AS `AND[((cast(id as DOUBLE) + random(2)) >= 0.2),((cast(id as DOUBLE) + random(2)) <= 0.22)]`, AND[((cast(t1.id as DOUBLE) + $_random_11_$) >= 0.3),((cast(t1.id as DOUBLE) + $_random_11_$) <= 0.33)] AS `AND[((cast(id as DOUBLE) + random(3)) >= 0.3),((cast(id as DOUBLE) + random(3)) <= 0.33)]`, cast(AND[((cast(t1.id as DOUBLE) + $_random_9_$) >= 0.1),((cast(t1.id as DOUBLE) + $_random_9_$) <= 0.11)] as TINYINT) AS `cast(AND[((cast(id as DOUBLE) + random(1)) >= 0.1),((cast(id as DOUBLE) + random(1)) <= 0.11)] as TINYINT)`] ----------PhysicalProject[random(1) AS `$_random_9_$`, random(2) AS `$_random_10_$`, random(3) AS `$_random_11_$`, t1.id] ------------PhysicalOlapScan[t1] @@ -182,9 +182,10 @@ PhysicalResultSink ----NestedLoopJoin[INNER_JOIN](((cast(id as BIGINT) + cast(id as BIGINT)) + $_random_9_$) >= 10)(((cast(id as BIGINT) + cast(id as BIGINT)) + $_random_9_$) <= 20) ------PhysicalProject[$_random_9_$, cast(id as BIGINT), t1.id, t1.msg] --------filter(($_random_11_$ <= 10) and ($_random_11_$ >= 1)) -----------PhysicalProject[cast(id as BIGINT) AS `cast(id as BIGINT)`, random(1, 100) AS `$_random_11_$`, random(1, 100) AS `$_random_9_$`, t1.id, t1.msg] +----------PhysicalProject[cast(t1.id as BIGINT) AS `cast(id as BIGINT)`, random(1, 100) AS `$_random_11_$`, random(1, 100) AS `$_random_9_$`, t1.id, t1.msg] ------------PhysicalOlapScan[t1] ------PhysicalProject[cast(id as BIGINT), t2.id, t2.msg] --------filter(((cast(id as BIGINT) * $_random_10_$) <= 200) and ((cast(id as BIGINT) * $_random_10_$) >= 100)) -----------PhysicalProject[cast(id as BIGINT) AS `cast(id as BIGINT)`, random(1, 100) AS `$_random_10_$`, t2.id, t2.msg] +----------PhysicalProject[cast(t2.id as BIGINT) AS `cast(id as BIGINT)`, random(1, 100) AS `$_random_10_$`, t2.id, t2.msg] ------------PhysicalOlapScan[t2] + diff --git a/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out index adc172f81d6a20..d8c8bbef72cb2b 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out @@ -5,42 +5,42 @@ PhysicalResultSink -- !check_equal_no_agg_2_shape -- PhysicalResultSink ---PhysicalProject[(a + random()) AS `a + random()`, (a + random()) AS `a + random()`] +--PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ----PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_no_agg_3_shape -- PhysicalResultSink ---PhysicalProject[(a + random()) AS `a + random()`, (a + random()) AS `a + random()`] +--PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ----filter(((tbl_unique_function_with_one_row.a + random()) > 0.01)) ------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_no_agg_4_shape -- PhysicalResultSink ---PhysicalProject[(a + random()) AS `a + random()`, (a + random()) AS `a + random()`, sum(a + random()) over(), sum(a + random()) over()] +--PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`, sum(a + random()) over(), sum(a + random()) over()] ----PhysicalWindow -------PhysicalProject[(a + random()) AS `(a + random())`, (a + random()) AS `(a + random())`, tbl_unique_function_with_one_row.a] +------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, (tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, tbl_unique_function_with_one_row.a] --------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_no_agg_5_shape -- PhysicalResultSink ---PhysicalProject[(a + random()) AS `a + random()`, (a + random()) AS `a + random()`] +--PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ----filter((sum((a + random())) OVER() > 0.01) and (sum((a + random())) OVER(PARTITION BY (a + random())) > 0.01)) ------PhysicalWindow --------PhysicalWindow ----------PhysicalQuickSort[LOCAL_SORT] -------------PhysicalProject[(a + random()) AS `(a + random())`, (a + random()) AS `(a + random())`, (a + random()) AS `(a + random())`, tbl_unique_function_with_one_row.a] +------------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, (tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, (tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, tbl_unique_function_with_one_row.a] --------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_no_agg_6_shape -- PhysicalResultSink ---PhysicalProject[(a + random()) AS `a + random()`, (a + random()) AS `a + random()`, sum(a + random()) over (partition by a + random())] +--PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`, sum(a + random()) over (partition by a + random())] ----filter((sum((a + random())) OVER() > 0.01) and (sum((a + random())) OVER(PARTITION BY (a + random())) > 0.01)) ------PhysicalWindow --------PhysicalWindow ----------PhysicalQuickSort[LOCAL_SORT] ------------PhysicalWindow --------------PhysicalQuickSort[LOCAL_SORT] -----------------PhysicalProject[(a + random()) AS `(a + random())`, (a + random()) AS `(a + random())`, (a + random()) AS `(a + random())`, (a + random()) AS `(a + random())`, (a + random()) AS `(a + random())`, tbl_unique_function_with_one_row.a] +----------------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, (tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, (tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, (tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, (tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, tbl_unique_function_with_one_row.a] ------------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_one_row_to_agg_1_shape -- @@ -102,7 +102,7 @@ PhysicalResultSink ----PhysicalQuickSort[LOCAL_SORT] ------hashAgg[GLOBAL] --------hashAgg[LOCAL] -----------PhysicalProject[(a + random()) AS `a + random()`, (a + random()) AS `a + random()`] +----------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_agg_with_groupby_1_shape -- @@ -135,7 +135,7 @@ PhysicalResultSink -- !check_equal_agg_with_groupby_4_shape -- PhysicalResultSink ---PhysicalProject[(a + random()) AS `a + random()`, (a + random()) AS `a + random()`, sum(a + random()), sum(a + random())] +--PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`, sum(a + random()), sum(a + random())] ----hashAgg[GLOBAL] ------hashAgg[LOCAL] --------PhysicalProject[tbl_unique_function_with_one_row.a] @@ -143,7 +143,7 @@ PhysicalResultSink -- !check_equal_agg_with_groupby_5_shape -- PhysicalResultSink ---PhysicalProject[(a + random()) AS `a + random()`, (a + random()) AS `a + random()`, sum(a + random()), sum(a + random())] +--PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`, sum(a + random()), sum(a + random())] ----filter(((tbl_unique_function_with_one_row.a + random()) > 0.01)) ------hashAgg[GLOBAL] --------hashAgg[LOCAL] @@ -155,7 +155,7 @@ PhysicalResultSink --PhysicalProject[a + random(), a + random(), abs(a + random()), sum(a + random()), sum(a + random())] ----PhysicalQuickSort[MERGE_SORT] ------PhysicalQuickSort[LOCAL_SORT] ---------PhysicalProject[(a + random()) AS `(a + random())`, (a + random()) AS `a + random()`, (a + random()) AS `a + random()`, abs((a + random())) AS `abs(a + random())`, sum(a + random()), sum(a + random())] +--------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `(a + random())`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`, abs((tbl_unique_function_with_one_row.a + random())) AS `abs(a + random())`, sum(a + random()), sum(a + random())] ----------filter(((tbl_unique_function_with_one_row.a + random()) > 0.01)) ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] @@ -168,7 +168,7 @@ PhysicalResultSink ----PhysicalWindow ------hashAgg[GLOBAL] --------hashAgg[LOCAL] -----------PhysicalProject[(a + random()) AS `a + random()`] +----------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_agg_with_groupby_8_shape -- @@ -178,7 +178,7 @@ PhysicalResultSink ------hashAgg[GLOBAL] --------hashAgg[LOCAL] ----------filter((a + random() > 0.01)) -------------PhysicalProject[(a + random()) AS `a + random()`] +------------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`] --------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_agg_with_groupby_9_shape -- @@ -190,7 +190,7 @@ PhysicalResultSink ----------hashAgg[GLOBAL] ------------hashAgg[LOCAL] --------------filter((a + random() > 0.01)) -----------------PhysicalProject[(a + random()) AS `a + random()`] +----------------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ------------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_agg_with_groupby_10_shape -- @@ -198,7 +198,7 @@ PhysicalResultSink --PhysicalProject[a + random(), a + random() AS `a + random()`, sum(a + random()) AS `sum(a + random())`, sum(a + random()) AS `sum(a + random())`] ----hashAgg[GLOBAL] ------hashAgg[LOCAL] ---------PhysicalProject[(a + random()) AS `a + random()`] +--------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ----------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_agg_with_groupby_11_shape -- @@ -207,7 +207,7 @@ PhysicalResultSink ----hashAgg[GLOBAL] ------hashAgg[LOCAL] --------filter((a + random() > 0.01)) -----------PhysicalProject[(a + random()) AS `a + random()`] +----------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_agg_with_groupby_12_shape -- @@ -218,7 +218,7 @@ PhysicalResultSink --------hashAgg[GLOBAL] ----------hashAgg[LOCAL] ------------filter((a + random() > 0.01)) ---------------PhysicalProject[(a + random()) AS `a + random()`] +--------------PhysicalProject[(tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ----------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_agg_with_groupby_13_shape -- @@ -226,7 +226,7 @@ PhysicalResultSink --PhysicalProject[a + random(), a + random() + 0, sum(a + random() + 0) AS `sum(a + random() + 0)`, sum(a + random()) AS `sum(a + random())`] ----hashAgg[GLOBAL] ------hashAgg[LOCAL] ---------PhysicalProject[((a + random()) + 0.0) AS `a + random() + 0`, (a + random()) AS `a + random()`] +--------PhysicalProject[((tbl_unique_function_with_one_row.a + random()) + 0.0) AS `a + random() + 0`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ----------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_agg_with_groupby_14_shape -- @@ -237,7 +237,7 @@ PhysicalResultSink --------hashAgg[GLOBAL] ----------hashAgg[LOCAL] ------------filter((a + random() > 0.01)) ---------------PhysicalProject[((a + random()) + 0.0) AS `a + random() + 0`, (a + random()) AS `a + random()`] +--------------PhysicalProject[((tbl_unique_function_with_one_row.a + random()) + 0.0) AS `a + random() + 0`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ----------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_agg_with_groupby_15_shape -- @@ -249,7 +249,7 @@ PhysicalResultSink ----------hashAgg[GLOBAL] ------------hashAgg[LOCAL] --------------filter((a + random() + 0 > 0.01)) -----------------PhysicalProject[((a + random()) + 0.0) AS `a + random() + 0`, (a + random()) AS `a + random()`] +----------------PhysicalProject[((tbl_unique_function_with_one_row.a + random()) + 0.0) AS `a + random() + 0`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ------------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_repeat1_shape -- @@ -258,13 +258,13 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----hashAgg[GLOBAL] ------hashAgg[LOCAL] --------PhysicalProject[a + random(), a + random() + 0, a + random() + 0 AS `a + random() + 0`, a + random() AS `a + random()`] -----------PhysicalProject[((a + random()) + 0.0) AS `a + random() + 0`, (a + random()) AS `a + random()`] +----------PhysicalProject[((tbl_unique_function_with_one_row.a + random()) + 0.0) AS `a + random() + 0`, (tbl_unique_function_with_one_row.a + random()) AS `a + random()`] ------------PhysicalOlapScan[tbl_unique_function_with_one_row] --PhysicalResultSink ----PhysicalProject[a + random(), a + random() + 0, abs(a + random()), sum(a + random() + 0), sum(a + random())] ------PhysicalQuickSort[MERGE_SORT] --------PhysicalQuickSort[LOCAL_SORT] -----------PhysicalProject[(a + random() + 1.0) AS `(a + random() + 1.0)`, a + random() + 0 AS `a + random() + 0`, a + random() AS `a + random()`, abs(a + random()) AS `abs(a + random())`, sum(a + random() + 0) AS `sum(a + random() + 0)`, sum(a + random()) AS `sum(a + random())`] +----------PhysicalProject[(.a + random() + 1.0) AS `(a + random() + 1.0)`, .a + random() + 0 AS `a + random() + 0`, .a + random() AS `a + random()`, .sum(a + random() + 0) AS `sum(a + random() + 0)`, .sum(a + random()) AS `sum(a + random())`, abs(.a + random()) AS `abs(a + random())`] ------------filter((.a + random() + 0 > 0.01)) --------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/nereids_rules_p0/unique_function/between_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/between_with_unique_function.out index 9e489a3bfb10c1..59ad2f53fa133b 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/between_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/between_with_unique_function.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !between_two_num -- PhysicalResultSink ---filter((cast(id as DOUBLE) <= random()) and (cast(id as DOUBLE) >= random())) +--filter((cast(t1.id as DOUBLE) <= random()) and (cast(t1.id as DOUBLE) >= random())) ----PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/unique_function/filter_through_project_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/filter_through_project_with_unique_function.out index 5547b5aa7d8d17..833e4dd00f3003 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/filter_through_project_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/filter_through_project_with_unique_function.out @@ -1,34 +1,34 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !filter_through_project_1 -- PhysicalResultSink ---PhysicalProject[(cast(id as BIGINT) + 100) AS `a`, (cast(id as BIGINT) + 200) AS `b`, (cast(id as BIGINT) + 300) AS `c`] +--PhysicalProject[(cast(t1.id as BIGINT) + 100) AS `a`, (cast(t1.id as BIGINT) + 200) AS `b`, (cast(t1.id as BIGINT) + 300) AS `c`] ----filter((t1.id > 899)) ------PhysicalOlapScan[t1] -- !filter_through_project_2 -- PhysicalResultSink --filter((t.a > 999)) -----PhysicalProject[((cast(id as BIGINT) + random(1, 10)) + 100) AS `a`, (cast(id as BIGINT) + 200) AS `b`, (cast(id as BIGINT) + 300) AS `c`] +----PhysicalProject[((cast(t1.id as BIGINT) + random(1, 10)) + 100) AS `a`, (cast(t1.id as BIGINT) + 200) AS `b`, (cast(t1.id as BIGINT) + 300) AS `c`] ------filter((t1.id > 799)) --------PhysicalOlapScan[t1] -- !filter_through_project_3 -- PhysicalResultSink --filter((t.a > 999) and (t.b > 999)) -----PhysicalProject[((cast(id as BIGINT) + random(1, 10)) + 100) AS `a`, ((cast(id as BIGINT) + random(1, 10)) + 200) AS `b`, ((cast(id as BIGINT) + random(1, 10)) + 300) AS `c`] +----PhysicalProject[((cast(t1.id as BIGINT) + random(1, 10)) + 100) AS `a`, ((cast(t1.id as BIGINT) + random(1, 10)) + 200) AS `b`, ((cast(t1.id as BIGINT) + random(1, 10)) + 300) AS `c`] ------PhysicalOlapScan[t1] -- !filter_through_project_4 -- PhysicalResultSink ---PhysicalProject[(cast(id as BIGINT) + 100) AS `a`, (cast(id as BIGINT) + 200) AS `b`, (cast(id as BIGINT) + 300) AS `c`] -----filter(((cast(id as BIGINT) + random(1, 10)) > 799) and ((cast(id as BIGINT) + random(1, 10)) > 899)) +--PhysicalProject[(cast(t1.id as BIGINT) + 100) AS `a`, (cast(t1.id as BIGINT) + 200) AS `b`, (cast(t1.id as BIGINT) + 300) AS `c`] +----filter(((cast(t1.id as BIGINT) + random(1, 10)) > 799) and ((cast(t1.id as BIGINT) + random(1, 10)) > 899)) ------PhysicalOlapScan[t1] -- !filter_through_project_5 -- PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] -------PhysicalProject[(cast(id as BIGINT) + 100) AS `a`, (cast(id as BIGINT) + 200) AS `b`, (cast(id as BIGINT) + 300) AS `c`] +------PhysicalProject[(cast(t1.id as BIGINT) + 100) AS `a`, (cast(t1.id as BIGINT) + 200) AS `b`, (cast(t1.id as BIGINT) + 300) AS `c`] --------filter((t1.id > 899)) ----------PhysicalOlapScan[t1] @@ -37,7 +37,7 @@ PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] ------filter((t.a > 999)) ---------PhysicalProject[((cast(id as BIGINT) + random(1, 10)) + 100) AS `a`, (cast(id as BIGINT) + 200) AS `b`, (cast(id as BIGINT) + 300) AS `c`] +--------PhysicalProject[((cast(t1.id as BIGINT) + random(1, 10)) + 100) AS `a`, (cast(t1.id as BIGINT) + 200) AS `b`, (cast(t1.id as BIGINT) + 300) AS `c`] ----------filter((t1.id > 799)) ------------PhysicalOlapScan[t1] @@ -46,14 +46,14 @@ PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] ------filter((t.a > 999) and (t.b > 999)) ---------PhysicalProject[((cast(id as BIGINT) + random(1, 10)) + 100) AS `a`, ((cast(id as BIGINT) + random(1, 10)) + 200) AS `b`, ((cast(id as BIGINT) + random(1, 10)) + 300) AS `c`] +--------PhysicalProject[((cast(t1.id as BIGINT) + random(1, 10)) + 100) AS `a`, ((cast(t1.id as BIGINT) + random(1, 10)) + 200) AS `b`, ((cast(t1.id as BIGINT) + random(1, 10)) + 300) AS `c`] ----------PhysicalOlapScan[t1] -- !filter_through_project_8 -- PhysicalResultSink --PhysicalLimit[GLOBAL] ----PhysicalLimit[LOCAL] -------PhysicalProject[(cast(id as BIGINT) + 100) AS `a`, (cast(id as BIGINT) + 200) AS `b`, (cast(id as BIGINT) + 300) AS `c`] ---------filter(((cast(id as BIGINT) + random(1, 10)) > 799) and ((cast(id as BIGINT) + random(1, 10)) > 899)) +------PhysicalProject[(cast(t1.id as BIGINT) + 100) AS `a`, (cast(t1.id as BIGINT) + 200) AS `b`, (cast(t1.id as BIGINT) + 300) AS `c`] +--------filter(((cast(t1.id as BIGINT) + random(1, 10)) > 799) and ((cast(t1.id as BIGINT) + random(1, 10)) > 899)) ----------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/unique_function/infer_predicates_set_op_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/infer_predicates_set_op_with_unique_function.out index 2b20e4a761a516..f3e36d733c0129 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/infer_predicates_set_op_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/infer_predicates_set_op_with_unique_function.out @@ -3,7 +3,7 @@ PhysicalResultSink --PhysicalExcept ----PhysicalProject[t1.id] -------filter(((cast(id as DOUBLE) + random()) > 5.0)) +------filter(((cast(t1.id as DOUBLE) + random()) > 5.0)) --------PhysicalOlapScan[t1] ----PhysicalProject[t2.id] ------PhysicalOlapScan[t2] @@ -12,7 +12,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalIntersect ----PhysicalProject[t1.id] -------filter(((cast(id as DOUBLE) + random()) > 5.0)) +------filter(((cast(t1.id as DOUBLE) + random()) > 5.0)) --------PhysicalOlapScan[t1] ----PhysicalProject[t2.id] ------PhysicalOlapScan[t2] @@ -21,7 +21,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalExcept ----PhysicalProject[t1.id] -------filter(((uuid_to_int(uuid()) + cast(id as LARGEINT)) > 5)) +------filter(((uuid_to_int(uuid()) + cast(t1.id as LARGEINT)) > 5)) --------PhysicalOlapScan[t1] ----PhysicalProject[t2.id] ------PhysicalOlapScan[t2] @@ -30,7 +30,7 @@ PhysicalResultSink PhysicalResultSink --PhysicalIntersect ----PhysicalProject[t1.id] -------filter(((uuid_to_int(uuid()) + cast(id as LARGEINT)) > 5)) +------filter(((uuid_to_int(uuid()) + cast(t1.id as LARGEINT)) > 5)) --------PhysicalOlapScan[t1] ----PhysicalProject[t2.id] ------PhysicalOlapScan[t2] diff --git a/regression-test/data/nereids_rules_p0/unique_function/merge_project_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/merge_project_with_unique_function.out index 2572bdaf39027c..75d8b51603c2e8 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/merge_project_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/merge_project_with_unique_function.out @@ -1,30 +1,30 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !merge_project_1 -- PhysicalResultSink ---PhysicalProject[(cast(id as BIGINT) + 100) AS `b`, (cast(id as BIGINT) + 100) AS `c`] +--PhysicalProject[(cast(t1.id as BIGINT) + 100) AS `b`, (cast(t1.id as BIGINT) + 100) AS `c`] ----PhysicalOlapScan[t1] -- !merge_project_2 -- PhysicalResultSink ---PhysicalProject[a AS `b`, a AS `c`] -----PhysicalProject[(cast(id as BIGINT) + random(1, 10)) AS `a`] +--PhysicalProject[t.a AS `b`, t.a AS `c`] +----PhysicalProject[(cast(t1.id as BIGINT) + random(1, 10)) AS `a`] ------PhysicalOlapScan[t1] -- !merge_project_3 -- PhysicalResultSink ---PhysicalProject[(cast(id as BIGINT) + random(1, 10)) AS `b`] +--PhysicalProject[(cast(t1.id as BIGINT) + random(1, 10)) AS `b`] ----PhysicalOlapScan[t1] -- !merge_project_4 -- PhysicalResultSink ---PhysicalProject[((a + a) + 10) AS `b`] -----PhysicalProject[(cast(id as BIGINT) + random(1, 10)) AS `a`] +--PhysicalProject[((t.a + t.a) + 10) AS `b`] +----PhysicalProject[(cast(t1.id as BIGINT) + random(1, 10)) AS `a`] ------PhysicalOlapScan[t1] -- !merge_project_5 -- PhysicalResultSink ---PhysicalProject[(a + 10) AS `c`, a AS `b`] -----PhysicalProject[(cast(id as BIGINT) + random(1, 10)) AS `a`] +--PhysicalProject[(t.a + 10) AS `c`, t.a AS `b`] +----PhysicalProject[(cast(t1.id as BIGINT) + random(1, 10)) AS `a`] ------PhysicalOlapScan[t1] -- !merge_one_row_1 -- @@ -33,7 +33,7 @@ PhysicalResultSink -- !merge_one_row_2 -- PhysicalResultSink ---PhysicalProject[a AS `b`, a AS `c`] +--PhysicalProject[t.a AS `b`, t.a AS `c`] ----PhysicalOneRowRelation[random(1, 10) AS `a`] -- !merge_one_row_3 -- @@ -42,12 +42,12 @@ PhysicalResultSink -- !merge_one_row_4 -- PhysicalResultSink ---PhysicalProject[((a + a) + 10) AS `b`] +--PhysicalProject[((t.a + t.a) + 10) AS `b`] ----PhysicalOneRowRelation[random(1, 10) AS `a`] -- !merge_one_row_5 -- PhysicalResultSink ---PhysicalProject[(a + 10) AS `c`, a AS `b`] +--PhysicalProject[(t.a + 10) AS `c`, t.a AS `b`] ----PhysicalOneRowRelation[random(1, 10) AS `a`] -- !merge_empty_1 -- @@ -72,35 +72,35 @@ PhysicalResultSink -- !merge_union_1 -- PhysicalResultSink ---PhysicalProject[a AS `b`, a AS `c`] +--PhysicalProject[t.a AS `b`, t.a AS `c`] ----PhysicalUnion(constantExprsList=[[100 AS `100`]]) -------PhysicalProject[(cast(id as BIGINT) + 100) AS `a`] +------PhysicalProject[(cast(t1.id as BIGINT) + 100) AS `a`] --------PhysicalOlapScan[t1] -- !merge_union_2 -- PhysicalResultSink ---PhysicalProject[a AS `b`, a AS `c`] +--PhysicalProject[t.a AS `b`, t.a AS `c`] ----PhysicalUnion(constantExprsList=[[random(1, 10) AS `random(1, 10)`]]) -------PhysicalProject[(cast(id as BIGINT) + random(1, 10)) AS `a`] +------PhysicalProject[(cast(t1.id as BIGINT) + random(1, 10)) AS `a`] --------PhysicalOlapScan[t1] -- !merge_union_3 -- PhysicalResultSink --PhysicalUnion(constantExprsList=[[random(1, 10) AS `b`]]) -----PhysicalProject[(cast(id as BIGINT) + random(1, 10)) AS `b`] +----PhysicalProject[(cast(t1.id as BIGINT) + random(1, 10)) AS `b`] ------PhysicalOlapScan[t1] -- !merge_union_4 -- PhysicalResultSink ---PhysicalProject[((a + a) + 10) AS `b`] +--PhysicalProject[((t.a + t.a) + 10) AS `b`] ----PhysicalUnion(constantExprsList=[[random(1, 10) AS `random(1, 10)`]]) -------PhysicalProject[(cast(id as BIGINT) + random(1, 10)) AS `a`] +------PhysicalProject[(cast(t1.id as BIGINT) + random(1, 10)) AS `a`] --------PhysicalOlapScan[t1] -- !merge_union_5 -- PhysicalResultSink ---PhysicalProject[(a + 10) AS `c`, a AS `b`] +--PhysicalProject[(t.a + 10) AS `c`, t.a AS `b`] ----PhysicalUnion(constantExprsList=[[random(1, 10) AS `random(1, 10)`]]) -------PhysicalProject[(cast(id as BIGINT) + random(1, 10)) AS `a`] +------PhysicalProject[(cast(t1.id as BIGINT) + random(1, 10)) AS `a`] --------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/unique_function/project_other_join_condition_for_nlj_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/project_other_join_condition_for_nlj_with_unique_function.out index 66c185735b7e2e..43045271a9860e 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/project_other_join_condition_for_nlj_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/project_other_join_condition_for_nlj_with_unique_function.out @@ -3,32 +3,32 @@ PhysicalResultSink --PhysicalProject[t1.id, t2.id] ----NestedLoopJoin[LEFT_OUTER_JOIN]((cast(id as DOUBLE) + random()) < cast(id as DOUBLE)) -------PhysicalProject[cast(id as DOUBLE) AS `cast(id as DOUBLE)`, t1.id] +------PhysicalProject[cast(t1.id as DOUBLE) AS `cast(id as DOUBLE)`, t1.id] --------PhysicalOlapScan[t1] -------PhysicalProject[cast(id as DOUBLE) AS `cast(id as DOUBLE)`, t2.id] +------PhysicalProject[cast(t2.id as DOUBLE) AS `cast(id as DOUBLE)`, t2.id] --------PhysicalOlapScan[t2] -- !cross_rand_one_side -- PhysicalResultSink --PhysicalProject[t1.id, t2.id] ----NestedLoopJoin[INNER_JOIN]((cast(id as DOUBLE) + random()) > cast(id as DOUBLE)) -------PhysicalProject[cast(id as DOUBLE) AS `cast(id as DOUBLE)`, t1.id] +------PhysicalProject[cast(t1.id as DOUBLE) AS `cast(id as DOUBLE)`, t1.id] --------PhysicalOlapScan[t1] -------PhysicalProject[cast(id as DOUBLE) AS `cast(id as DOUBLE)`, t2.id] +------PhysicalProject[cast(t2.id as DOUBLE) AS `cast(id as DOUBLE)`, t2.id] --------PhysicalOlapScan[t2] -- !cross_rand_both_sides -- PhysicalResultSink --PhysicalProject[t1.id, t2.id] ----NestedLoopJoin[INNER_JOIN]((cast(id as DOUBLE) + random()) > (cast(id as DOUBLE) + random())) -------PhysicalProject[cast(id as DOUBLE) AS `cast(id as DOUBLE)`, t1.id] +------PhysicalProject[cast(t1.id as DOUBLE) AS `cast(id as DOUBLE)`, t1.id] --------PhysicalOlapScan[t1] -------PhysicalProject[cast(id as DOUBLE) AS `cast(id as DOUBLE)`, t2.id] +------PhysicalProject[cast(t2.id as DOUBLE) AS `cast(id as DOUBLE)`, t2.id] --------PhysicalOlapScan[t2] -- !hash_and_other_with_volatile_equal -- PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((cast(id as BIGINT) = (cast(id as BIGINT) + random(1, 10)))) +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((cast(t1.id as BIGINT) = (cast(t2.id as BIGINT) + random(1, 10)))) ----PhysicalProject[t1.id] ------PhysicalOlapScan[t1] ----PhysicalProject[t2.id] diff --git a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_agg_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_agg_with_unique_function.out index 94f7250b53f8b4..fcd52cd5909874 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_agg_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_agg_with_unique_function.out @@ -1,9 +1,9 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !push_down_filter_through_agg -- PhysicalResultSink ---filter(((cast(id as BIGINT) + random(1, 10)) > 6)) +--filter(((cast(t1.id as BIGINT) + random(1, 10)) > 6)) ----hashAgg[GLOBAL] ------PhysicalProject[t1.id] ---------filter(((cast(id as BIGINT) + random(1, 10)) > 5)) +--------filter(((cast(t1.id as BIGINT) + random(1, 10)) > 5)) ----------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_generate_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_generate_with_unique_function.out index a2506f5d8a7673..9a6c27ac520e0b 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_generate_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_generate_with_unique_function.out @@ -10,7 +10,7 @@ PhysicalResultSink -- !filter_through_generate_unique_2 -- PhysicalResultSink --PhysicalProject[tmp1.e1] -----filter(((cast(id as BIGINT) + random(1, 100)) > 5)) +----filter(((cast(t1.id as BIGINT) + random(1, 100)) > 5)) ------PhysicalGenerate --------PhysicalProject[t1.id] ----------filter((t1.id > 10)) @@ -19,7 +19,7 @@ PhysicalResultSink -- !filter_through_generate_unique_3 -- PhysicalResultSink --PhysicalProject[tmp1.e1] -----filter(((cast(id as BIGINT) + random(1, 100)) > 5)) +----filter(((cast(t1.id as BIGINT) + random(1, 100)) > 5)) ------PhysicalGenerate --------PhysicalProject[t1.id] ----------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_join_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_join_with_unique_function.out index c3902875ecb140..d1942c652980b6 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_join_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_join_with_unique_function.out @@ -13,15 +13,15 @@ PhysicalResultSink --PhysicalProject[t1.id, t2.id] ----filter((random() > 0.1)) ------hashJoin[INNER_JOIN] hashCondition=((expr_cast(id as BIGINT) = expr_(cast(id as BIGINT) + 10))) otherCondition=() ---------PhysicalProject[cast(id as BIGINT) AS `expr_cast(id as BIGINT)`, t1.id] +--------PhysicalProject[cast(t1.id as BIGINT) AS `expr_cast(id as BIGINT)`, t1.id] ----------filter((t1.id > 100)) ------------PhysicalOlapScan[t1] ---------PhysicalProject[(cast(id as BIGINT) + 10) AS `expr_(cast(id as BIGINT) + 10)`, t2.id] +--------PhysicalProject[(cast(t2.id as BIGINT) + 10) AS `expr_(cast(id as BIGINT) + 10)`, t2.id] ----------PhysicalOlapScan[t2] -- !push_down_filter_through_join_3 -- PhysicalResultSink ---filter(((cast(id as BIGINT) + random(1, 100)) > 100)) +--filter(((cast(t1.id as BIGINT) + random(1, 100)) > 100)) ----NestedLoopJoin[CROSS_JOIN] ------PhysicalProject[t1.id] --------PhysicalOlapScan[t1] @@ -31,22 +31,22 @@ PhysicalResultSink -- !reorder_join_1 -- PhysicalResultSink --PhysicalProject[t1.id, t2.id] -----filter(((cast(id as BIGINT) + random(1, 100)) = cast(id as BIGINT))) +----filter(((cast(t1.id as BIGINT) + random(1, 100)) = cast(t2.id as BIGINT))) ------hashJoin[INNER_JOIN] hashCondition=((expr_(cast(id as BIGINT) * 2) = expr_(cast(id as BIGINT) * 5))) otherCondition=() ---------PhysicalProject[(cast(id as BIGINT) * 2) AS `expr_(cast(id as BIGINT) * 2)`, t1.id] +--------PhysicalProject[(cast(t1.id as BIGINT) * 2) AS `expr_(cast(id as BIGINT) * 2)`, t1.id] ----------PhysicalOlapScan[t1] ---------PhysicalProject[(cast(id as BIGINT) * 5) AS `expr_(cast(id as BIGINT) * 5)`, t2.id] +--------PhysicalProject[(cast(t2.id as BIGINT) * 5) AS `expr_(cast(id as BIGINT) * 5)`, t2.id] ----------PhysicalOlapScan[t2] -- !reorder_join_2 -- PhysicalResultSink ---filter(((cast(id as BIGINT) + random(1, 100)) = cast(id as BIGINT))) +--filter(((cast(t1.id as BIGINT) + random(1, 100)) = cast(t3.id as BIGINT))) ----NestedLoopJoin[CROSS_JOIN] ------PhysicalProject[t1.id, t2.id] --------hashJoin[INNER_JOIN] hashCondition=((expr_(cast(id as BIGINT) * 2) = expr_(cast(id as BIGINT) * 5))) otherCondition=() -----------PhysicalProject[(cast(id as BIGINT) * 2) AS `expr_(cast(id as BIGINT) * 2)`, t1.id] +----------PhysicalProject[(cast(t1.id as BIGINT) * 2) AS `expr_(cast(id as BIGINT) * 2)`, t1.id] ------------PhysicalOlapScan[t1] -----------PhysicalProject[(cast(id as BIGINT) * 5) AS `expr_(cast(id as BIGINT) * 5)`, t2.id] +----------PhysicalProject[(cast(t2.id as BIGINT) * 5) AS `expr_(cast(id as BIGINT) * 5)`, t2.id] ------------PhysicalOlapScan[t2] ------PhysicalProject[t3.id] --------PhysicalOlapScan[t2(t3)] @@ -56,18 +56,18 @@ PhysicalResultSink --PhysicalProject[t1.id, t2.id, t3.id] ----filter((random() > 10.0)) ------hashJoin[INNER_JOIN] hashCondition=((expr_(cast(id as BIGINT) * 2) = expr_(cast(id as BIGINT) * 5))) otherCondition=() ---------PhysicalProject[(cast(id as BIGINT) * 2) AS `expr_(cast(id as BIGINT) * 2)`, t1.id, t2.id] +--------PhysicalProject[(cast(t1.id as BIGINT) * 2) AS `expr_(cast(id as BIGINT) * 2)`, t1.id, t2.id] ----------NestedLoopJoin[CROSS_JOIN] ------------PhysicalProject[t1.id] --------------PhysicalOlapScan[t1] ------------PhysicalProject[t2.id] --------------PhysicalOlapScan[t2] ---------PhysicalProject[(cast(id as BIGINT) * 5) AS `expr_(cast(id as BIGINT) * 5)`, t3.id] +--------PhysicalProject[(cast(t3.id as BIGINT) * 5) AS `expr_(cast(id as BIGINT) * 5)`, t3.id] ----------PhysicalOlapScan[t2(t3)] -- !push_down_filter_through_join_4 -- PhysicalResultSink ---filter(((cast(id as DOUBLE) + random()) > 0.2) and (random() > 0.1)) +--filter(((cast(t2.id as DOUBLE) + random()) > 0.2) and (random() > 0.1)) ----NestedLoopJoin[CROSS_JOIN] ------PhysicalProject[t1.id] --------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_partition_topn_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_partition_topn_with_unique_function.out index e7b584cc57d026..c5902425b1eb50 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_partition_topn_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_partition_topn_with_unique_function.out @@ -9,7 +9,7 @@ PhysicalResultSink -- !filter_through_partition_topn_unique_2 -- PhysicalResultSink ---filter(((cast(id as BIGINT) + random(1, 100)) > 5) and (v.rn <= 3)) +--filter(((cast(v.id as BIGINT) + random(1, 100)) > 5) and (v.rn <= 3)) ----PhysicalWindow ------PhysicalQuickSort[LOCAL_SORT] --------PhysicalPartitionTopN diff --git a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_window_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_window_with_unique_function.out index a7083b712d7a90..868108f06576d4 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_window_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/push_down_filter_through_window_with_unique_function.out @@ -16,7 +16,7 @@ PhysicalResultSink -- !filter_through_window_unique_3 -- PhysicalResultSink ---filter(((cast(id as BIGINT) + random(1, 100)) > 5)) +--filter(((cast(v.id as BIGINT) + random(1, 100)) > 5)) ----PhysicalWindow ------PhysicalQuickSort[LOCAL_SORT] --------PhysicalOlapScan[t1] diff --git a/regression-test/data/nereids_rules_p0/unique_function/push_down_join_other_condition_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/push_down_join_other_condition_with_unique_function.out index 79b10152c1304d..4281be944fce84 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/push_down_join_other_condition_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/push_down_join_other_condition_with_unique_function.out @@ -21,7 +21,7 @@ PhysicalResultSink -- !join_other_unique_4 -- PhysicalResultSink ---hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=(((cast(id as DOUBLE) + random()) > 0.5)) +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=(((cast(t1.id as DOUBLE) + random()) > 0.5)) ----PhysicalOlapScan[t1] ----PhysicalOlapScan[t2] @@ -29,7 +29,7 @@ PhysicalResultSink PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ----PhysicalProject[t1.id, t1.msg] -------filter(((cast(id as DOUBLE) + $_random_5_$) <= 0.5) and ((cast(id as DOUBLE) + $_random_5_$) >= 0.1)) +------filter(((cast(t1.id as DOUBLE) + $_random_5_$) <= 0.5) and ((cast(t1.id as DOUBLE) + $_random_5_$) >= 0.1)) --------PhysicalProject[random() AS `$_random_5_$`, t1.id, t1.msg] ----------PhysicalOlapScan[t1] ----PhysicalOlapScan[t2] @@ -39,7 +39,7 @@ PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ----PhysicalOlapScan[t1] ----PhysicalProject[t2.id, t2.msg] -------filter(((cast(id as DOUBLE) + $_random_5_$) <= 0.5) and ((cast(id as DOUBLE) + $_random_5_$) >= 0.1)) +------filter(((cast(t2.id as DOUBLE) + $_random_5_$) <= 0.5) and ((cast(t2.id as DOUBLE) + $_random_5_$) >= 0.1)) --------PhysicalProject[random() AS `$_random_5_$`, t2.id, t2.msg] ----------PhysicalOlapScan[t2] @@ -51,3 +51,4 @@ PhysicalResultSink --------PhysicalProject[random() AS `$_random_5_$`, t1.id, t1.msg] ----------PhysicalOlapScan[t1] ----PhysicalOlapScan[t2] + diff --git a/regression-test/data/nereids_rules_p0/unique_function/push_project_into_union_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/push_project_into_union_with_unique_function.out index 7af0a74a43a4b0..924934dd2be7ab 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/push_project_into_union_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/push_project_into_union_with_unique_function.out @@ -13,16 +13,16 @@ PhysicalResultSink -- !no_push_down_1 -- PhysicalResultSink ---PhysicalProject[a AS `b`, a AS `c`] +--PhysicalProject[t.a AS `b`, t.a AS `c`] ----PhysicalUnion(constantExprsList=[[random() AS `a`], [200.0 AS `a`]]) -- !no_push_down_2 -- PhysicalResultSink ---PhysicalProject[((a + a) + 10.0) AS `b`] +--PhysicalProject[((t.a + t.a) + 10.0) AS `b`] ----PhysicalUnion(constantExprsList=[[random() AS `a`], [200.0 AS `a`]]) -- !no_push_down_3 -- PhysicalResultSink ---PhysicalProject[(a + 10.0) AS `c`, a AS `b`] +--PhysicalProject[(t.a + 10.0) AS `c`, t.a AS `b`] ----PhysicalUnion(constantExprsList=[[random() AS `a`], [200.0 AS `a`]]) diff --git a/regression-test/data/nereids_syntax_p0/analyze_agg.out b/regression-test/data/nereids_syntax_p0/analyze_agg.out index 18bd2adc2fd216..b286120c880f44 100644 --- a/regression-test/data/nereids_syntax_p0/analyze_agg.out +++ b/regression-test/data/nereids_syntax_p0/analyze_agg.out @@ -59,7 +59,7 @@ PhysicalResultSink --------------filter((sum((cast(id as BIGINT) + random(1, 1))) > 1)) ----------------hashAgg[GLOBAL] ------------------PhysicalProject ---------------------filter(((cast(id as BIGINT) + random(1, 1)) > 0) and (t1.__DORIS_DELETE_SIGN__ = 0)) +--------------------filter(((cast(t1.id as BIGINT) + random(1, 1)) > 0) and (t1.__DORIS_DELETE_SIGN__ = 0)) ----------------------PhysicalOlapScan[t1] -- !having_with_window_2 -- @@ -75,7 +75,7 @@ PhysicalResultSink ------------------filter((sum((cast(id as BIGINT) + random(1, 100))) > 1)) --------------------hashAgg[GLOBAL] ----------------------PhysicalProject -------------------------filter(((cast(id as BIGINT) + random(1, 100)) > 0) and (t1.__DORIS_DELETE_SIGN__ = 0)) +------------------------filter(((cast(t1.id as BIGINT) + random(1, 100)) > 0) and (t1.__DORIS_DELETE_SIGN__ = 0)) --------------------------PhysicalOlapScan[t1] -- !having_with_window_3 -- diff --git a/regression-test/data/query_p0/cache/query_cache_incremental.out b/regression-test/data/query_p0/cache/query_cache_incremental.out new file mode 100644 index 00000000000000..f1314984593fb9 --- /dev/null +++ b/regression-test/data/query_p0/cache/query_cache_incremental.out @@ -0,0 +1,29 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dup_initial -- +/a 75 3 +/b 20 1 +/c 135 3 + +-- !dup_after_appends -- +/a 130 13 +/b 20 1 +/c 135 3 +/inc 550 10 + +-- !dup_final -- +/a 79 11 +/after-delete 1 1 +/b 20 1 +/c 135 3 +/inc 550 10 + +-- !mow_initial -- +2026-01-01 10 +2026-01-02 20 +2026-01-06 30 + +-- !mow_final -- +2026-01-01 100 +2026-01-02 20 +2026-01-06 43 + diff --git a/regression-test/data/query_p0/eager_agg/eager_agg.out b/regression-test/data/query_p0/eager_agg/eager_agg.out index cb991cf56ebb14..476aad842425ce 100644 --- a/regression-test/data/query_p0/eager_agg/eager_agg.out +++ b/regression-test/data/query_p0/eager_agg/eager_agg.out @@ -378,7 +378,7 @@ PhysicalResultSink -- !check_filter_slots_preserved_pushdown -- PhysicalResultSink --hashAgg[GLOBAL] -----filter(OR[( not (id = 1)),id IS NULL]) +----filter(OR[( not (b.id = 1)),b.id IS NULL]) ------hashJoin[LEFT_OUTER_JOIN] hashCondition=((a.id = b.id)) otherCondition=() --------hashAgg[GLOBAL] ----------PhysicalOlapScan[eager_agg_filter_t1(a)] diff --git a/regression-test/data/query_p0/eager_agg/scalar_agg_pushdown.out b/regression-test/data/query_p0/eager_agg/scalar_agg_pushdown.out new file mode 100644 index 00000000000000..8fa4341a049603 --- /dev/null +++ b/regression-test/data/query_p0/eager_agg/scalar_agg_pushdown.out @@ -0,0 +1,41 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !scalar_agg_no_pushdown -- + +-- !const_group_key_shape -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------NestedLoopJoin[CROSS_JOIN] +--------NestedLoopJoin[INNER_JOIN] +----------filter((ta.a = 999)) +------------PhysicalOlapScan[t1(ta)] +----------filter((tb.a = 999)) +------------PhysicalOlapScan[t1(tb)] +--------PhysicalStorageLayerAggregate[t2] + +Hint log: +Used: leading(ta tb ) +UnUsed: +SyntaxError: + +-- !const_group_key_exe -- + +-- !pushdown_preserved_shape -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------NestedLoopJoin[CROSS_JOIN] +--------hashJoin[INNER_JOIN] hashCondition=((ta.a = tb.a)) otherCondition=() +----------hashAgg[GLOBAL] +------------PhysicalOlapScan[t1(ta)] +----------PhysicalOlapScan[t1(tb)] +--------PhysicalStorageLayerAggregate[t2] + +Hint log: +Used: leading(ta broadcast tb ) +UnUsed: +SyntaxError: + +-- !pushdown_preserved_exe -- +1 60 + diff --git a/regression-test/data/query_p0/eliminate_outer_join/eliminate_outer_join.out b/regression-test/data/query_p0/eliminate_outer_join/eliminate_outer_join.out index 2449cf55b94d5f..6a94a1cf1001aa 100644 --- a/regression-test/data/query_p0/eliminate_outer_join/eliminate_outer_join.out +++ b/regression-test/data/query_p0/eliminate_outer_join/eliminate_outer_join.out @@ -5,11 +5,11 @@ PhysicalResultSink ----hashJoin[INNER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ------hashJoin[INNER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() --------PhysicalOlapScan[region] ---------filter(( not n_nationkey IS NULL) and (nation.n_nationkey > 1)) +--------filter(( not nation.n_nationkey IS NULL) and (nation.n_nationkey > 1)) ----------PhysicalOlapScan[nation] -------filter(( not s_suppkey IS NULL) and (supplier.s_suppkey > 1)) +------filter(( not supplier.s_suppkey IS NULL) and (supplier.s_suppkey > 1)) --------PhysicalOlapScan[supplier] -----filter(( not ps_suppkey IS NULL) and (partsupp.ps_suppkey > 1)) +----filter(( not partsupp.ps_suppkey IS NULL) and (partsupp.ps_suppkey > 1)) ------PhysicalOlapScan[partsupp] Hint log: @@ -26,7 +26,7 @@ PhysicalResultSink ----------PhysicalOlapScan[region] ----------PhysicalOlapScan[nation] --------PhysicalOlapScan[supplier] -----filter(( not ps_suppkey IS NULL) and (partsupp.ps_suppkey > 1)) +----filter(( not partsupp.ps_suppkey IS NULL) and (partsupp.ps_suppkey > 1)) ------PhysicalOlapScan[partsupp] Hint log: @@ -43,7 +43,7 @@ PhysicalResultSink ----------PhysicalOlapScan[region] ----------PhysicalOlapScan[nation] --------PhysicalOlapScan[supplier] -----filter(( not ps_suppkey IS NULL) and (partsupp.ps_suppkey > 1)) +----filter(( not partsupp.ps_suppkey IS NULL) and (partsupp.ps_suppkey > 1)) ------PhysicalOlapScan[partsupp] Hint log: @@ -54,7 +54,7 @@ SyntaxError: -- !4 -- PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() -----filter(( not r_name IS NULL) and (length(r_name) = 0)) +----filter(( not region.r_name IS NULL) and (length(region.r_name) = 0)) ------PhysicalOlapScan[region] ----PhysicalOlapScan[nation] @@ -67,7 +67,7 @@ SyntaxError: PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() -------filter(( not r_name IS NULL) and (length(r_name) = 0)) +------filter(( not region.r_name IS NULL) and (length(region.r_name) = 0)) --------PhysicalOlapScan[region] ------PhysicalOlapScan[nation] ----PhysicalOlapScan[supplier] @@ -82,7 +82,7 @@ PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((partsupp.ps_suppkey = supplier.s_suppkey)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ------hashJoin[LEFT_OUTER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() ---------filter(( not r_name IS NULL) and (length(r_name) = 0)) +--------filter(( not region.r_name IS NULL) and (length(region.r_name) = 0)) ----------PhysicalOlapScan[region] --------PhysicalOlapScan[nation] ------PhysicalOlapScan[supplier] @@ -98,9 +98,9 @@ PhysicalResultSink --hashJoin[FULL_OUTER_JOIN] hashCondition=((partsupp.ps_suppkey = supplier.s_suppkey)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ------hashJoin[INNER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() ---------filter(( not r_regionkey IS NULL)) +--------filter(( not region.r_regionkey IS NULL)) ----------PhysicalOlapScan[region] ---------filter(( not n_regionkey IS NULL)) +--------filter(( not nation.n_regionkey IS NULL)) ----------PhysicalOlapScan[nation] ------PhysicalOlapScan[supplier] ----PhysicalOlapScan[partsupp] @@ -115,9 +115,9 @@ PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((partsupp.ps_suppkey = supplier.s_suppkey)) otherCondition=() ----hashJoin[LEFT_OUTER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ------hashJoin[INNER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() ---------filter(( not r_name IS NULL) and ( not r_regionkey IS NULL) and (length(r_name) = 0)) +--------filter(( not region.r_name IS NULL) and ( not region.r_regionkey IS NULL) and (length(region.r_name) = 0)) ----------PhysicalOlapScan[region] ---------filter(( not n_regionkey IS NULL)) +--------filter(( not nation.n_regionkey IS NULL)) ----------PhysicalOlapScan[nation] ------PhysicalOlapScan[supplier] ----PhysicalOlapScan[partsupp] @@ -132,9 +132,9 @@ PhysicalResultSink --hashJoin[LEFT_OUTER_JOIN] hashCondition=((partsupp.ps_suppkey = supplier.s_suppkey)) otherCondition=() ----hashJoin[INNER_JOIN] hashCondition=((nation.n_nationkey = supplier.s_suppkey)) otherCondition=() ------hashJoin[INNER_JOIN] hashCondition=((region.r_regionkey = nation.n_regionkey)) otherCondition=() ---------filter(( not r_name IS NULL) and ( not r_regionkey IS NULL) and (length(r_name) = 0)) +--------filter(( not region.r_name IS NULL) and ( not region.r_regionkey IS NULL) and (length(region.r_name) = 0)) ----------PhysicalOlapScan[region] ---------filter(( not n_regionkey IS NULL)) +--------filter(( not nation.n_regionkey IS NULL)) ----------PhysicalOlapScan[nation] ------PhysicalOlapScan[supplier] ----PhysicalOlapScan[partsupp] diff --git a/regression-test/data/query_p0/hint/fix_leading.out b/regression-test/data/query_p0/hint/fix_leading.out index 73abf90eb058de..526c6eac8ea93b 100644 --- a/regression-test/data/query_p0/hint/fix_leading.out +++ b/regression-test/data/query_p0/hint/fix_leading.out @@ -4,7 +4,7 @@ PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3) and (t1.c1 = t4.c4)) otherCondition=() ----NestedLoopJoin[CROSS_JOIN] ------PhysicalOlapScan[t1] -------filter(( not c2 IS NULL)) +------filter(( not t2.c2 IS NULL)) --------PhysicalOlapScan[t2] ----hashJoin[INNER_JOIN] hashCondition=((t3.c3 = t4.c4)) otherCondition=() ------PhysicalOlapScan[t3] diff --git a/regression-test/data/query_p0/physical_translator/test_physical_translator.out b/regression-test/data/query_p0/physical_translator/test_physical_translator.out index 1bc6c15f7a4d35..f161f191d91d0b 100644 --- a/regression-test/data/query_p0/physical_translator/test_physical_translator.out +++ b/regression-test/data/query_p0/physical_translator/test_physical_translator.out @@ -1,17 +1,17 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !continue_project_shape -- PhysicalResultSink ---PhysicalProject[t.a, x AS `k1`, x AS `k2`] +--PhysicalProject[t.a, t.x AS `k1`, t.x AS `k2`] ----PhysicalProject[random(100) AS `x`, tbl_test_physical_translator.a] ------PhysicalOlapScan[tbl_test_physical_translator] -- !continue_project_result -- 1 0.9616644308453555 0.9616644308453555 -200 0.32358402105146866 0.32358402105146866 +200 0.3235840210514687 0.3235840210514687 -- !continue_filter_shape -- PhysicalResultSink ---PhysicalProject[(cast(a as BIGINT) + 2) AS `x`] +--PhysicalProject[(cast(tbl_test_physical_translator.a as BIGINT) + 2) AS `x`] ----filter((tbl_test_physical_translator.a < 9998) and (tbl_test_physical_translator.a > 3)) ------PhysicalLimit[GLOBAL] --------PhysicalLimit[LOCAL] diff --git a/regression-test/data/query_p0/runtime_filter/rf_partition_pruning.out b/regression-test/data/query_p0/runtime_filter/rf_partition_pruning.out index 44295ef30a19b8..3fe042541fb594 100644 --- a/regression-test/data/query_p0/runtime_filter/rf_partition_pruning.out +++ b/regression-test/data/query_p0/runtime_filter/rf_partition_pruning.out @@ -136,5 +136,11 @@ Beijing 3 5 3 e 6 3 f +-- !rf_prune_expr_without_partition_prune -- +1 + +-- !rf_prune_expr_with_partition_prune -- +1 + -- !list_str_bloom -- 3 c diff --git a/regression-test/data/query_p0/set_operations/bucket_shuffle_set_operation.out b/regression-test/data/query_p0/set_operations/bucket_shuffle_set_operation.out index 50a3c5cc131ef6..1ec75925ec2da0 100644 --- a/regression-test/data/query_p0/set_operations/bucket_shuffle_set_operation.out +++ b/regression-test/data/query_p0/set_operations/bucket_shuffle_set_operation.out @@ -117,3 +117,188 @@ PhysicalResultSink 2 3 +-- !bucket_shuffle_join_as_basic_child_shape -- +PhysicalResultSink +--PhysicalIntersect[bucketShuffle] +----PhysicalProject +------hashJoin[INNER_JOIN broadcast] hashCondition=((a.id = b.id)) otherCondition=() +--------PhysicalProject +----------PhysicalOlapScan[bucket_shuffle_set_operation1(a)] +--------PhysicalProject +----------PhysicalOlapScan[bucket_shuffle_set_operation2(b)] +----PhysicalDistribute[DistributionSpecHash] +------PhysicalProject +--------PhysicalOlapScan[bucket_shuffle_set_operation3] + +-- !bucket_shuffle_join_as_basic_child_result -- +1 +2 +3 + +-- !bucket_shuffle_nested_set_operation_shape -- +PhysicalResultSink +--PhysicalUnion +----PhysicalDistribute[DistributionSpecExecutionAny] +------PhysicalProject +--------PhysicalOlapScan[bucket_shuffle_set_operation3] +----PhysicalDistribute[DistributionSpecExecutionAny] +------PhysicalIntersect[bucketShuffle] +--------PhysicalProject +----------hashJoin[INNER_JOIN broadcast] hashCondition=((a.id = b.id)) otherCondition=() +------------PhysicalProject +--------------PhysicalOlapScan[bucket_shuffle_set_operation1(a)] +------------PhysicalProject +--------------PhysicalOlapScan[bucket_shuffle_set_operation2(b)] +--------PhysicalDistribute[DistributionSpecHash] +----------PhysicalProject +------------PhysicalOlapScan[bucket_shuffle_set_operation2] + +-- !bucket_shuffle_nested_set_operation_result -- +1 +1 +2 +2 +3 +3 + +-- !bucket_shuffle_when_local_shuffle_off_shape -- +PhysicalResultSink +--PhysicalIntersect[bucketShuffle] +----PhysicalProject +------PhysicalOlapScan[bucket_shuffle_set_operation1] +----PhysicalDistribute[DistributionSpecHash] +------PhysicalProject +--------PhysicalOlapScan[bucket_shuffle_set_operation2] + +-- !bucket_shuffle_when_local_shuffle_off_result -- +1 +2 +3 + +-- !union_parent_hash_shape_when_local_shuffle_planner_off -- +PhysicalResultSink +--PhysicalProject +----hashJoin[INNER_JOIN bucketShuffle] hashCondition=((u.id = b.id)) otherCondition=() +------PhysicalUnion +--------PhysicalDistribute[DistributionSpecHash] +----------PhysicalProject +------------PhysicalOlapScan[bucket_shuffle_set_operation1] +--------PhysicalDistribute[DistributionSpecHash] +----------PhysicalProject +------------PhysicalOlapScan[bucket_shuffle_set_operation2] +------PhysicalProject +--------PhysicalOlapScan[bucket_shuffle_set_operation3(b)] + +Hint log: +Used: [shuffle]_1 +UnUsed: +SyntaxError: + +-- !union_parent_hash_when_local_shuffle_planner_off -- +1 +1 +2 +2 +3 +3 + +-- !plain_intersect_when_local_shuffle_planner_off -- +1 +2 +3 + +-- !plain_intersect_when_nereids_distribute_planner_off -- +1 +2 +3 + +-- !union_parent_hash_when_nereids_distribute_planner_off -- +1 +1 +2 +2 +3 +3 + +-- !intersect_right_basic_parent_hash -- +1 +2 +3 + +-- !bucket_shuffle_union_fill_up -- +0 +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +2 +3 +4 +4 +5 +6 +6 +7 +8 +9 + +-- !bucket_shuffle_equivalent_key_fill_up -- +0 +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +2 +3 +4 +5 +6 +7 +8 +9 + +-- !bucket_shuffle_two_equivalent_keys -- +0 0 +1 1 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +2 2 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 + +-- !multi_column_bucket_key_intersect -- +1 +2 +3 + diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_of_array_difference.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_of_array_difference.out index afdad5f9500e8a..5cec2b9c19e4a1 100644 --- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_of_array_difference.out +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions_of_array_difference.out @@ -7,6 +7,9 @@ 5 [16, 7, 8] [0, -9, 1] 6 [1, 2, 3, 4, 5, 4, 3, 2, 1] [0, 1, 1, 1, 1, -1, -1, -1, -1] 7 [1111, 12324, 8674, 123, 3434, 435, 45, 53, 54, 2] [0, 11213, -3650, -8551, 3311, -2999, -390, 8, 1, -52] +8 [42] [0] +9 [2147483647, -2147483648] [0, -4294967295] +10 [-2147483648, 2147483647] [0, 4294967295] -- !select -- [0, 1, 1, 1] @@ -14,3 +17,12 @@ -- !select -- [0, 6, 93, -95] +-- !select -- +[0] + +-- !select -- +[0, -4294967295] + +-- !select -- +[0, 4294967295] + diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_nested_array_map.out b/regression-test/data/query_p0/sql_functions/array_functions/test_nested_array_map.out new file mode 100644 index 00000000000000..32404ea64dcd3b --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_nested_array_map.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 [5, 7, 9] +2 [11, 22, 3] + +-- !select2 -- +[[-9, -8], [-19, -18]] + +-- !select_nested_array_sortby -- +[[1, 3, 5, 9], [5, 3, 9, 1], [9, 5, 3, 1]] + +-- !select_same_name_shadow -- +[[2, 3], [4, 5]] diff --git a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out index a162f381a4af7e..8fa306872e2176 100644 --- a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out +++ b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out @@ -47,6 +47,27 @@ true -- !sql -- false +-- !sql -- +false + +-- !sql -- +true + +-- !sql -- +true + +-- !sql -- +true + +-- !sql -- +false + +-- !sql -- +true + +-- !sql -- +false + -- !sql_bitmap_hash1 -- 1 diff --git a/regression-test/data/query_p0/sql_functions/json_functions/test_json_function.out b/regression-test/data/query_p0/sql_functions/json_functions/test_json_function.out index 21129e5247a0f3..6a69d5c87123d9 100644 --- a/regression-test/data/query_p0/sql_functions/json_functions/test_json_function.out +++ b/regression-test/data/query_p0/sql_functions/json_functions/test_json_function.out @@ -164,20 +164,11 @@ true \N 2 -- !sql -- -"doris" - --- !sql -- -\N +doris -- !sql -- \N --- !sql -- -["v1",{"k21":6.6,"k22":[1,2,3]}] - --- !sql -- -[6.6,[1,2,3],2] - -- !sql -- 123 diff --git a/regression-test/data/query_p0/virtual_slot_ref/adjust_virtual_slot_nullable.out b/regression-test/data/query_p0/virtual_slot_ref/adjust_virtual_slot_nullable.out index 7785b1127f64c5..9c774760a897f2 100644 --- a/regression-test/data/query_p0/virtual_slot_ref/adjust_virtual_slot_nullable.out +++ b/regression-test/data/query_p0/virtual_slot_ref/adjust_virtual_slot_nullable.out @@ -4,7 +4,7 @@ PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((t1.c_int = t2.c_int)) otherCondition=() ----PhysicalOlapScan[tbl_adjust_virtual_slot_nullable_1(t1)] ----PhysicalProject -------filter(OR[( not dayofmonth(c_date) IN (1, 3)),( not (cast(dayofmonth(c_date) as INT) = c_int))]) +------filter(OR[( not dayofmonth(c_date) IN (1, 3)),( not (cast(dayofmonth(c_date) as INT) = t2.c_int))]) --------PhysicalOlapScan[tbl_adjust_virtual_slot_nullable_2(t2)] -- !left_join_result -- diff --git a/regression-test/data/shape_check/clickbench/query11.out b/regression-test/data/shape_check/clickbench/query11.out index 0fac14167ad449..5a2a783c63156e 100644 --- a/regression-test/data/shape_check/clickbench/query11.out +++ b/regression-test/data/shape_check/clickbench/query11.out @@ -9,6 +9,6 @@ PhysicalResultSink ------------hashAgg[LOCAL] --------------hashAgg[GLOBAL] ----------------PhysicalProject -------------------filter(( not (length(MobilePhoneModel) = 0))) +------------------filter(( not (length(hits.MobilePhoneModel) = 0))) --------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query12.out b/regression-test/data/shape_check/clickbench/query12.out index 69ad0325529bde..76cf04f442a54e 100644 --- a/regression-test/data/shape_check/clickbench/query12.out +++ b/regression-test/data/shape_check/clickbench/query12.out @@ -9,6 +9,6 @@ PhysicalResultSink ------------hashAgg[LOCAL] --------------hashAgg[GLOBAL] ----------------PhysicalProject -------------------filter(( not (length(MobilePhoneModel) = 0))) +------------------filter(( not (length(hits.MobilePhoneModel) = 0))) --------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query13.out b/regression-test/data/shape_check/clickbench/query13.out index 0856fc73a84030..3dd9d9982f192c 100644 --- a/regression-test/data/shape_check/clickbench/query13.out +++ b/regression-test/data/shape_check/clickbench/query13.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (length(SearchPhrase) = 0))) +----------------filter(( not (length(hits.SearchPhrase) = 0))) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query14.out b/regression-test/data/shape_check/clickbench/query14.out index a33268c31e4623..18cda012b89938 100644 --- a/regression-test/data/shape_check/clickbench/query14.out +++ b/regression-test/data/shape_check/clickbench/query14.out @@ -9,6 +9,6 @@ PhysicalResultSink ------------hashAgg[LOCAL] --------------hashAgg[GLOBAL] ----------------PhysicalProject -------------------filter(( not (length(SearchPhrase) = 0))) +------------------filter(( not (length(hits.SearchPhrase) = 0))) --------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query15.out b/regression-test/data/shape_check/clickbench/query15.out index a2eb22e6f2b532..533f322c55e45a 100644 --- a/regression-test/data/shape_check/clickbench/query15.out +++ b/regression-test/data/shape_check/clickbench/query15.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (length(SearchPhrase) = 0))) +----------------filter(( not (length(hits.SearchPhrase) = 0))) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query2.out b/regression-test/data/shape_check/clickbench/query2.out index 4f4565a083c67b..85cd133ac0ef47 100644 --- a/regression-test/data/shape_check/clickbench/query2.out +++ b/regression-test/data/shape_check/clickbench/query2.out @@ -5,6 +5,6 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------filter(( not (AdvEngineID = 0))) +----------filter(( not (hits.AdvEngineID = 0))) ------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query21.out b/regression-test/data/shape_check/clickbench/query21.out index 104d1b4710532a..270e1c945abb83 100644 --- a/regression-test/data/shape_check/clickbench/query21.out +++ b/regression-test/data/shape_check/clickbench/query21.out @@ -5,6 +5,6 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------filter((URL like '%google%')) +----------filter((hits.URL like '%google%')) ------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query22.out b/regression-test/data/shape_check/clickbench/query22.out index 02b2d803778be1..5e42c588274af9 100644 --- a/regression-test/data/shape_check/clickbench/query22.out +++ b/regression-test/data/shape_check/clickbench/query22.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (length(SearchPhrase) = 0)) and (URL like '%google%')) +----------------filter(( not (length(hits.SearchPhrase) = 0)) and (hits.URL like '%google%')) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query23.out b/regression-test/data/shape_check/clickbench/query23.out index fec234f0fffd80..018616ec9cf1ca 100644 --- a/regression-test/data/shape_check/clickbench/query23.out +++ b/regression-test/data/shape_check/clickbench/query23.out @@ -9,6 +9,6 @@ PhysicalResultSink ------------hashAgg[LOCAL] --------------hashAgg[GLOBAL] ----------------PhysicalProject -------------------filter(( not (URL like '%.google.%')) and ( not (length(SearchPhrase) = 0)) and (Title like '%Google%')) +------------------filter(( not (hits.URL like '%.google.%')) and ( not (length(hits.SearchPhrase) = 0)) and (hits.Title like '%Google%')) --------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query24.out b/regression-test/data/shape_check/clickbench/query24.out index 5aebbbfaa49ff5..37b696e2ad7db0 100644 --- a/regression-test/data/shape_check/clickbench/query24.out +++ b/regression-test/data/shape_check/clickbench/query24.out @@ -6,6 +6,6 @@ PhysicalResultSink ------PhysicalTopN[MERGE_SORT] --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] -------------filter((URL like '%google%')) +------------filter((hits.URL like '%google%')) --------------PhysicalLazyMaterializeOlapScan[hits lazySlots:(hits.CounterID,hits.EventDate,hits.UserID,hits.WatchID,hits.JavaEnable,hits.Title,hits.GoodEvent,hits.ClientIP,hits.RegionID,hits.CounterClass,hits.OS,hits.UserAgent,hits.Referer,hits.IsRefresh,hits.RefererCategoryID,hits.RefererRegionID,hits.URLCategoryID,hits.URLRegionID,hits.ResolutionWidth,hits.ResolutionHeight,hits.ResolutionDepth,hits.FlashMajor,hits.FlashMinor,hits.FlashMinor2,hits.NetMajor,hits.NetMinor,hits.UserAgentMajor,hits.UserAgentMinor,hits.CookieEnable,hits.JavascriptEnable,hits.IsMobile,hits.MobilePhone,hits.MobilePhoneModel,hits.Params,hits.IPNetworkID,hits.TraficSourceID,hits.SearchEngineID,hits.SearchPhrase,hits.AdvEngineID,hits.IsArtifical,hits.WindowClientWidth,hits.WindowClientHeight,hits.ClientTimeZone,hits.ClientEventTime,hits.SilverlightVersion1,hits.SilverlightVersion2,hits.SilverlightVersion3,hits.SilverlightVersion4,hits.PageCharset,hits.CodeVersion,hits.IsLink,hits.IsDownload,hits.IsNotBounce,hits.FUniqID,hits.OriginalURL,hits.HID,hits.IsOldCounter,hits.IsEvent,hits.IsParameter,hits.DontCountHits,hits.WithHash,hits.HitColor,hits.LocalEventTime,hits.Age,hits.Sex,hits.Income,hits.Interests,hits.Robotness,hits.RemoteIP,hits.WindowName,hits.OpenerName,hits.HistoryLength,hits.BrowserLanguage,hits.BrowserCountry,hits.SocialNetwork,hits.SocialAction,hits.HTTPError,hits.SendTiming,hits.DNSTiming,hits.ConnectTiming,hits.ResponseStartTiming,hits.ResponseEndTiming,hits.FetchTiming,hits.SocialSourceNetworkID,hits.SocialSourcePage,hits.ParamPrice,hits.ParamOrderID,hits.ParamCurrency,hits.ParamCurrencyID,hits.OpenstatServiceName,hits.OpenstatCampaignID,hits.OpenstatAdID,hits.OpenstatSourceID,hits.UTMSource,hits.UTMMedium,hits.UTMCampaign,hits.UTMContent,hits.UTMTerm,hits.FromTag,hits.HasGCLID,hits.RefererHash,hits.URLHash,hits.CLID)] diff --git a/regression-test/data/shape_check/clickbench/query25.out b/regression-test/data/shape_check/clickbench/query25.out index 436a325ed91d88..04ef370e20817e 100644 --- a/regression-test/data/shape_check/clickbench/query25.out +++ b/regression-test/data/shape_check/clickbench/query25.out @@ -6,6 +6,6 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (length(SearchPhrase) = 0))) +------------filter(( not (length(hits.SearchPhrase) = 0))) --------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query26.out b/regression-test/data/shape_check/clickbench/query26.out index 373df683ec473e..c3aad84e48cd62 100644 --- a/regression-test/data/shape_check/clickbench/query26.out +++ b/regression-test/data/shape_check/clickbench/query26.out @@ -5,6 +5,6 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(( not (length(SearchPhrase) = 0))) +----------filter(( not (length(hits.SearchPhrase) = 0))) ------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query27.out b/regression-test/data/shape_check/clickbench/query27.out index e551d03fc48b41..195454389503aa 100644 --- a/regression-test/data/shape_check/clickbench/query27.out +++ b/regression-test/data/shape_check/clickbench/query27.out @@ -6,6 +6,6 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (length(SearchPhrase) = 0))) +------------filter(( not (length(hits.SearchPhrase) = 0))) --------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query28.out b/regression-test/data/shape_check/clickbench/query28.out index 7b07ad4defb38d..33ef91ba289124 100644 --- a/regression-test/data/shape_check/clickbench/query28.out +++ b/regression-test/data/shape_check/clickbench/query28.out @@ -9,6 +9,6 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------filter(( not (length(URL) = 0))) +------------------filter(( not (length(hits.URL) = 0))) --------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query29.out b/regression-test/data/shape_check/clickbench/query29.out index d1ed207fcdb070..d2e49e1edc6d0e 100644 --- a/regression-test/data/shape_check/clickbench/query29.out +++ b/regression-test/data/shape_check/clickbench/query29.out @@ -9,6 +9,6 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------filter(( not (length(Referer) = 0))) +------------------filter(( not (length(hits.Referer) = 0))) --------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query31.out b/regression-test/data/shape_check/clickbench/query31.out index fc6a57d2e1ec1e..7fcddcff8ad951 100644 --- a/regression-test/data/shape_check/clickbench/query31.out +++ b/regression-test/data/shape_check/clickbench/query31.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (length(SearchPhrase) = 0))) +----------------filter(( not (length(hits.SearchPhrase) = 0))) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query32.out b/regression-test/data/shape_check/clickbench/query32.out index 60bbde67814354..698817e9179fe4 100644 --- a/regression-test/data/shape_check/clickbench/query32.out +++ b/regression-test/data/shape_check/clickbench/query32.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (length(SearchPhrase) = 0))) +----------------filter(( not (length(hits.SearchPhrase) = 0))) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query37.out b/regression-test/data/shape_check/clickbench/query37.out index 58405571e1386a..e3445694c86c77 100644 --- a/regression-test/data/shape_check/clickbench/query37.out +++ b/regression-test/data/shape_check/clickbench/query37.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (length(URL) = 0)) and (hits.CounterID = 62) and (hits.DontCountHits = 0) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0)) +----------------filter(( not (length(hits.URL) = 0)) and (hits.CounterID = 62) and (hits.DontCountHits = 0) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0)) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query38.out b/regression-test/data/shape_check/clickbench/query38.out index a7bb448d79997c..c20e5907740d9d 100644 --- a/regression-test/data/shape_check/clickbench/query38.out +++ b/regression-test/data/shape_check/clickbench/query38.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (length(Title) = 0)) and (hits.CounterID = 62) and (hits.DontCountHits = 0) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0)) +----------------filter(( not (length(hits.Title) = 0)) and (hits.CounterID = 62) and (hits.DontCountHits = 0) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0)) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query39.out b/regression-test/data/shape_check/clickbench/query39.out index 89222c0f0d2abe..33685deb4f5586 100644 --- a/regression-test/data/shape_check/clickbench/query39.out +++ b/regression-test/data/shape_check/clickbench/query39.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (IsLink = 0)) and (hits.CounterID = 62) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsDownload = 0) and (hits.IsRefresh = 0)) +----------------filter(( not (hits.IsLink = 0)) and (hits.CounterID = 62) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsDownload = 0) and (hits.IsRefresh = 0)) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query41.out b/regression-test/data/shape_check/clickbench/query41.out index 8a7019e5969e79..c5f997387a155a 100644 --- a/regression-test/data/shape_check/clickbench/query41.out +++ b/regression-test/data/shape_check/clickbench/query41.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter((hits.CounterID = 62) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0) and (hits.RefererHash = 3594120000172545465) and TraficSourceID IN (-1, 6)) +----------------filter((hits.CounterID = 62) and (hits.EventDate <= '2013-07-31') and (hits.EventDate >= '2013-07-01') and (hits.IsRefresh = 0) and (hits.RefererHash = 3594120000172545465) and hits.TraficSourceID IN (-1, 6)) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/clickbench/query8.out b/regression-test/data/shape_check/clickbench/query8.out index 5bebd9361a2ca6..8a1331e18fcdf9 100644 --- a/regression-test/data/shape_check/clickbench/query8.out +++ b/regression-test/data/shape_check/clickbench/query8.out @@ -8,6 +8,6 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(( not (AdvEngineID = 0))) +----------------filter(( not (hits.AdvEngineID = 0))) ------------------PhysicalOlapScan[hits] diff --git a/regression-test/data/shape_check/ssb_sf100/shape/q3.3.out b/regression-test/data/shape_check/ssb_sf100/shape/q3.3.out index 2d43de3ad11a20..4432f41d297f2e 100644 --- a/regression-test/data/shape_check/ssb_sf100/shape/q3.3.out +++ b/regression-test/data/shape_check/ssb_sf100/shape/q3.3.out @@ -16,10 +16,10 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[lineorder] apply RFs: RF0 RF1 RF2 --------------------------PhysicalProject -----------------------------filter(s_city IN ('UNITED KI1', 'UNITED KI5')) +----------------------------filter(supplier.s_city IN ('UNITED KI1', 'UNITED KI5')) ------------------------------PhysicalOlapScan[supplier] ----------------------PhysicalProject -------------------------filter(c_city IN ('UNITED KI1', 'UNITED KI5')) +------------------------filter(customer.c_city IN ('UNITED KI1', 'UNITED KI5')) --------------------------PhysicalOlapScan[customer] ------------------PhysicalProject --------------------filter((dates.d_year <= 1997) and (dates.d_year >= 1992)) diff --git a/regression-test/data/shape_check/ssb_sf100/shape/q3.4.out b/regression-test/data/shape_check/ssb_sf100/shape/q3.4.out index b82e9655c20e9a..4600b2205356a0 100644 --- a/regression-test/data/shape_check/ssb_sf100/shape/q3.4.out +++ b/regression-test/data/shape_check/ssb_sf100/shape/q3.4.out @@ -16,10 +16,10 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[lineorder] apply RFs: RF0 RF1 RF2 --------------------------PhysicalProject -----------------------------filter(s_city IN ('UNITED KI1', 'UNITED KI5')) +----------------------------filter(supplier.s_city IN ('UNITED KI1', 'UNITED KI5')) ------------------------------PhysicalOlapScan[supplier] ----------------------PhysicalProject -------------------------filter(c_city IN ('UNITED KI1', 'UNITED KI5')) +------------------------filter(customer.c_city IN ('UNITED KI1', 'UNITED KI5')) --------------------------PhysicalOlapScan[customer] ------------------PhysicalProject --------------------filter((dates.d_yearmonth = 'Dec1997')) diff --git a/regression-test/data/shape_check/ssb_sf100/shape/q4.1.out b/regression-test/data/shape_check/ssb_sf100/shape/q4.1.out index 7ae89727caf6e3..0477c7aa496823 100644 --- a/regression-test/data/shape_check/ssb_sf100/shape/q4.1.out +++ b/regression-test/data/shape_check/ssb_sf100/shape/q4.1.out @@ -24,7 +24,7 @@ PhysicalResultSink ----------------------------filter((customer.c_region = 'AMERICA')) ------------------------------PhysicalOlapScan[customer] ----------------------PhysicalProject -------------------------filter(p_mfgr IN ('MFGR#1', 'MFGR#2')) +------------------------filter(part.p_mfgr IN ('MFGR#1', 'MFGR#2')) --------------------------PhysicalOlapScan[part] ------------------PhysicalProject --------------------PhysicalOlapScan[dates] diff --git a/regression-test/data/shape_check/ssb_sf100/shape/q4.2.out b/regression-test/data/shape_check/ssb_sf100/shape/q4.2.out index 34860dde1879ec..cd651ff80d089b 100644 --- a/regression-test/data/shape_check/ssb_sf100/shape/q4.2.out +++ b/regression-test/data/shape_check/ssb_sf100/shape/q4.2.out @@ -21,12 +21,12 @@ PhysicalResultSink --------------------------------filter((supplier.s_region = 'AMERICA')) ----------------------------------PhysicalOlapScan[supplier] --------------------------PhysicalProject -----------------------------filter(d_year IN (1997, 1998)) +----------------------------filter(dates.d_year IN (1997, 1998)) ------------------------------PhysicalOlapScan[dates] ----------------------PhysicalProject ------------------------filter((customer.c_region = 'AMERICA')) --------------------------PhysicalOlapScan[customer] ------------------PhysicalProject ---------------------filter(p_mfgr IN ('MFGR#1', 'MFGR#2')) +--------------------filter(part.p_mfgr IN ('MFGR#1', 'MFGR#2')) ----------------------PhysicalOlapScan[part] diff --git a/regression-test/data/shape_check/ssb_sf100/shape/q4.3.out b/regression-test/data/shape_check/ssb_sf100/shape/q4.3.out index 806229580b7cec..afdc70be66c0a3 100644 --- a/regression-test/data/shape_check/ssb_sf100/shape/q4.3.out +++ b/regression-test/data/shape_check/ssb_sf100/shape/q4.3.out @@ -26,6 +26,6 @@ PhysicalResultSink ----------------------------filter((part.p_category = 'MFGR#14')) ------------------------------PhysicalOlapScan[part] ----------------------PhysicalProject -------------------------filter(d_year IN (1997, 1998)) +------------------------filter(dates.d_year IN (1997, 1998)) --------------------------PhysicalOlapScan[dates] diff --git a/regression-test/data/shape_check/tpcds_sf100/constraints/query23.out b/regression-test/data/shape_check/tpcds_sf100/constraints/query23.out index 67ee2f066b0760..b6c70b2729f27a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/constraints/query23.out +++ b/regression-test/data/shape_check/tpcds_sf100/constraints/query23.out @@ -14,7 +14,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------PhysicalProject -------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[item] @@ -27,7 +27,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------filter(( not ss_customer_sk IS NULL)) +--------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------PhysicalOlapScan[store_sales] ----------PhysicalProject ------------hashAgg[GLOBAL] @@ -40,10 +40,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ------------------------------PhysicalProject ---------------------------------filter(( not ss_customer_sk IS NULL)) +--------------------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ------------------------------PhysicalProject ---------------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query1.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query1.out index e726e2ed72ec73..02c77571dc7b23 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query1.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query1.out @@ -18,7 +18,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) +------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->ctr_store_sk ------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query10.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query10.out index 75cbc2cb7f3471..0bce77d51ca68a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query10.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query10.out @@ -35,7 +35,7 @@ PhysicalResultSink ----------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalOlapScan[customer_demographics] --------------------------PhysicalProject -----------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County')) +----------------------------filter(ca.ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County')) ------------------------------PhysicalOlapScan[customer_address(ca)] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->cs_sold_date_sk diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query11.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query11.out index 4595f2f7447531..82b2bd291a180a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query11.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query11.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000) > if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000))) build RFs:RF11 customer_id->c_customer_id;RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(38, 8)) / t_w_firstyear.year_total), 0.000000) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(38, 8)) / t_s_firstyear.year_total), 0.000000))) build RFs:RF11 customer_id->c_customer_id;RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF9 customer_id->c_customer_id;RF10 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query12.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query12.out index dcf5a48b347927..3fcbb69e60cbef 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query12.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query12.out @@ -17,7 +17,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 ----------------------------PhysicalProject -------------------------------filter(i_category IN ('Books', 'Men', 'Sports')) +------------------------------filter(item.i_category IN ('Books', 'Men', 'Sports')) --------------------------------PhysicalOlapScan[item] ------------------------PhysicalProject --------------------------filter((date_dim.d_date <= '1998-05-06') and (date_dim.d_date >= '1998-04-06')) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query13.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query13.out index c1031d6e7a80d7..7c5e334687f3fe 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query13.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query13.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->ss_sold_date_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->ss_addr_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->ss_addr_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->ss_hdemo_sk --------------------PhysicalProject @@ -20,13 +20,13 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')]] and customer_demographics.cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and customer_demographics.cd_marital_status IN ('D', 'M', 'S')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter(hd_dep_count IN (1, 3)) +----------------------filter(household_demographics.hd_dep_count IN (1, 3)) ------------------------PhysicalOlapScan[household_demographics] ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'KS', 'MI', 'MO', 'ND', 'NH', 'OH', 'SD', 'TX')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('CO', 'KS', 'MI', 'MO', 'ND', 'NH', 'OH', 'SD', 'TX')) --------------------PhysicalOlapScan[customer_address] ------------PhysicalProject --------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query15.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query15.out index ecdec4af56925b..8f8ffe24167de6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query15.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query15.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),customer_address.ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() --------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query16.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query16.out index a77fc751be12e3..c5ddb58c3b1730 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query16.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query16.out @@ -16,7 +16,7 @@ PhysicalResultSink --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_ship_date_sk ----------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=() ------------------------------PhysicalProject ---------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF0 cs_order_number->cs_order_number +--------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF0 cs_order_number->cs_order_number ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF0 ----------------------------------PhysicalProject @@ -30,6 +30,6 @@ PhysicalResultSink --------------------------filter((customer_address.ca_state = 'WV')) ----------------------------PhysicalOlapScan[customer_address] --------------------PhysicalProject -----------------------filter(cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County')) +----------------------filter(call_center.cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County')) ------------------------PhysicalOlapScan[call_center] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query17.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query17.out index d4c0808a4bc2b3..db3c9d9a745965 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query17.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query17.out @@ -34,10 +34,10 @@ PhysicalResultSink ----------------------------------filter((d1.d_quarter_name = '2001Q1')) ------------------------------------PhysicalOlapScan[date_dim(d1)] ----------------------------PhysicalProject -------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +------------------------------filter(d2.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) --------------------------------PhysicalOlapScan[date_dim(d2)] ------------------------PhysicalProject ---------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------filter(d3.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------PhysicalOlapScan[date_dim(d3)] --------------------PhysicalProject ----------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query18.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query18.out index 50c94f23c06745..cf5db62ddf7ff7 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query18.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query18.out @@ -27,12 +27,12 @@ PhysicalResultSink --------------------------------------------filter((cd1.cd_education_status = 'Advanced Degree') and (cd1.cd_gender = 'F')) ----------------------------------------------PhysicalOlapScan[customer_demographics(cd1)] --------------------------------------PhysicalProject -----------------------------------------filter(c_birth_month IN (1, 10, 2, 4, 7, 8)) +----------------------------------------filter(customer.c_birth_month IN (1, 10, 2, 4, 7, 8)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF3 ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer_demographics(cd2)] ------------------------------PhysicalProject ---------------------------------filter(ca_state IN ('GA', 'IN', 'ME', 'NC', 'OK', 'WA', 'WY')) +--------------------------------filter(customer_address.ca_state IN ('GA', 'IN', 'ME', 'NC', 'OK', 'WA', 'WY')) ----------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query19.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query19.out index a664974b1303ea..b76fb2c6b7a942 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query19.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ss_item_sk ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query20.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query20.out index 8fc1e09860c453..9a59577c26c34e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query20.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query20.out @@ -17,7 +17,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 ----------------------------PhysicalProject -------------------------------filter(i_category IN ('Books', 'Shoes', 'Women')) +------------------------------filter(item.i_category IN ('Books', 'Shoes', 'Women')) --------------------------------PhysicalOlapScan[item] ------------------------PhysicalProject --------------------------filter((date_dim.d_date <= '2002-02-25') and (date_dim.d_date >= '2002-01-26')) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query21.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query21.out index 0e7fb61d00ffbc..6f4a6179ea1849 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query21.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query21.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) <= 1.5) and (if((inv_before > 0), (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) +--------filter(((cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)) <= 1.5) and (if((x.inv_before > 0), (cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query23.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query23.out index 6dc2bbbfcc8afd..b1922109307a90 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query23.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query23.out @@ -16,7 +16,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[item] ------------------PhysicalProject ---------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------PhysicalOlapScan[date_dim] --PhysicalCteAnchor ( cteId=CTEId#2 ) ----PhysicalCteProducer ( cteId=CTEId#2 ) @@ -27,7 +27,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------filter(( not ss_customer_sk IS NULL)) +--------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------PhysicalOlapScan[store_sales] ----------PhysicalProject ------------hashAgg[GLOBAL] @@ -40,10 +40,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ------------------------------PhysicalProject ---------------------------------filter(( not ss_customer_sk IS NULL)) +--------------------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ------------------------------PhysicalProject ---------------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query24.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query24.out index 506b9ebcb61a9b..88180fc27f6fc1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query24.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query24.out @@ -9,7 +9,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF6 s_zip->ca_zip;RF7 s_store_sk->ss_store_sk ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF5 ca_address_sk->c_current_addr_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (customer.c_birth_country = upper(customer_address.ca_country)))) build RFs:RF5 ca_address_sk->c_current_addr_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->ss_customer_sk ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query27.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query27.out index 8671f88820300a..920830ba9a65bd 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query27.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query27.out @@ -28,6 +28,6 @@ PhysicalResultSink ----------------------------filter((date_dim.d_year = 1999)) ------------------------------PhysicalOlapScan[date_dim] ----------------------PhysicalProject -------------------------filter(s_state IN ('AL', 'LA', 'MI', 'MO', 'SC', 'TN')) +------------------------filter(store.s_state IN ('AL', 'LA', 'MI', 'MO', 'SC', 'TN')) --------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query29.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query29.out index 53e3c00c17163f..b699a53258fecf 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query29.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query29.out @@ -36,7 +36,7 @@ PhysicalResultSink ----------------------------filter((d2.d_moy <= 7) and (d2.d_moy >= 4) and (d2.d_year = 1999)) ------------------------------PhysicalOlapScan[date_dim(d2)] ----------------------PhysicalProject -------------------------filter(d_year IN (1999, 2000, 2001)) +------------------------filter(d3.d_year IN (1999, 2000, 2001)) --------------------------PhysicalOlapScan[date_dim(d3)] ------------------PhysicalProject --------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query30.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query30.out index 0517776b8c6ffd..634cb8e81a5a3f 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query30.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query30.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) +----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query31.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query31.out index 8a42e70780102f..36b1a658693630 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query31.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query31.out @@ -15,7 +15,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject -------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +------------------filter((ss.d_year = 2000) and ss.d_qoy IN (1, 2, 3)) --------------------PhysicalOlapScan[date_dim] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) @@ -32,16 +32,16 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer_address] ------------------PhysicalProject ---------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) +--------------------filter((ws.d_year = 2000) and ws.d_qoy IN (1, 2, 3)) ----------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalQuickSort[MERGE_SORT] --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalQuickSort[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF13 ca_county->ca_county;RF14 ca_county->ca_county;RF15 ca_county->ca_county;RF16 ca_county->ca_county +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((ws2.web_sales > 0.00), (cast(ws3.web_sales as DECIMALV3(38, 8)) / ws2.web_sales), NULL) > if((ss2.store_sales > 0.00), (cast(ss3.store_sales as DECIMALV3(38, 8)) / ss2.store_sales), NULL))) build RFs:RF13 ca_county->ca_county;RF14 ca_county->ca_county;RF15 ca_county->ca_county;RF16 ca_county->ca_county ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF9 ca_county->ca_county;RF10 ca_county->ca_county;RF11 ca_county->ca_county +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((ws1.web_sales > 0.00), (cast(ws2.web_sales as DECIMALV3(38, 8)) / ws1.web_sales), NULL) > if((ss1.store_sales > 0.00), (cast(ss2.store_sales as DECIMALV3(38, 8)) / ss1.store_sales), NULL))) build RFs:RF9 ca_county->ca_county;RF10 ca_county->ca_county;RF11 ca_county->ca_county --------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss1.ca_county = ws1.ca_county)) otherCondition=() build RFs:RF6 ca_county->ca_county;RF7 ca_county->ca_county;RF12 ca_county->ca_county;RF17 ca_county->ca_county ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ss2.ca_county = ss3.ca_county)) otherCondition=() build RFs:RF5 ca_county->ca_county diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query32.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query32.out index ef8807bfdbf3ef..e252239b8dfac2 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query32.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query32.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +--------------filter((cast(catalog_sales.cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) ----------------PhysicalWindow ------------------PhysicalQuickSort[LOCAL_SORT] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query34.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query34.out index 109be4dafb4ae1..05f23d6e450ebe 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query34.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query34.out @@ -19,13 +19,13 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000)) +----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County')) +------------------------------filter(store.s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('0-500', '1001-5000')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('0-500', '1001-5000')) ----------------------------PhysicalOlapScan[household_demographics] ------------PhysicalProject --------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query36.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query36.out index 515d56d6252e70..cb6889a1558754 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query36.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query36.out @@ -28,6 +28,6 @@ PhysicalResultSink --------------------------------------filter((d1.d_year = 2002)) ----------------------------------------PhysicalOlapScan[date_dim(d1)] --------------------------------PhysicalProject -----------------------------------filter(s_state IN ('AL', 'GA', 'MI', 'MO', 'OH', 'SC', 'SD', 'TN')) +----------------------------------filter(store.s_state IN ('AL', 'GA', 'MI', 'MO', 'OH', 'SC', 'SD', 'TN')) ------------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query37.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query37.out index e2448337a82e48..7122ceb72487d6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query37.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query37.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856)) +----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and item.i_manufact_id IN (1000, 707, 747, 856)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21')) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query39.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query39.out index 2bfc7915a34000..bc074eeac089eb 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query39.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter(( not (mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) +------filter(( not (foo.mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) --------hashAgg[GLOBAL] ----------PhysicalProject ------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->inv_date_sk @@ -17,7 +17,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalProject --------------------PhysicalOlapScan[warehouse] --------------PhysicalProject -----------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2)) +----------------filter((date_dim.d_year = 1998) and inv.d_moy IN (1, 2)) ------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query4.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query4.out index 088261b87a10f0..0311e1eae66716 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query4.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query4.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_w_firstyear.year_total > 0.000000), (cast(t_w_secyear.year_total as DECIMALV3(38, 16)) / t_w_firstyear.year_total), NULL))) build RFs:RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF18 customer_id->c_customer_id;RF19 customer_id->c_customer_id;RF20 customer_id->c_customer_id;RF21 customer_id->c_customer_id ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF15 customer_id->c_customer_id;RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.000000), (cast(t_s_secyear.year_total as DECIMALV3(38, 16)) / t_s_firstyear.year_total), NULL))) build RFs:RF15 customer_id->c_customer_id;RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF12 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query41.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query41.out index c7c720dd75259e..12dad98df5b0fc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query41.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query41.out @@ -18,6 +18,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'medium'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('dodger', 'indian', 'spring', 'tan'),i_units IN ('Bunch', 'Carton', 'Tsp', 'Unknown'),OR[AND[i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp')],AND[i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown')]]],AND[i_size IN ('N/A', 'large'),i_color IN ('blue', 'chartreuse', 'peru', 'saddle'),i_units IN ('Each', 'Gram', 'Oz', 'Pallet'),OR[AND[i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz')],AND[i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet')]]]]],AND[(item.i_category = 'Women'),i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('aquamarine', 'blanched', 'gainsboro', 'tomato'),i_units IN ('Case', 'Dozen', 'Ounce', 'Tbl'),OR[AND[i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce')],AND[i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl')]]],AND[i_size IN ('extra large', 'small'),i_color IN ('almond', 'chiffon', 'lime', 'violet'),i_units IN ('Box', 'Dram', 'Pound', 'Ton'),OR[AND[i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton')],AND[i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram')]]]]]] and i_category IN ('Men', 'Women')) +------------------------------filter(OR[AND[(item.i_category = 'Men'),item.i_size IN ('N/A', 'economy', 'large', 'medium'),OR[AND[item.i_size IN ('economy', 'medium'),item.i_color IN ('dodger', 'indian', 'spring', 'tan'),item.i_units IN ('Bunch', 'Carton', 'Tsp', 'Unknown'),OR[AND[item.i_color IN ('dodger', 'tan'),item.i_units IN ('Bunch', 'Tsp')],AND[item.i_color IN ('indian', 'spring'),item.i_units IN ('Carton', 'Unknown')]]],AND[item.i_size IN ('N/A', 'large'),item.i_color IN ('blue', 'chartreuse', 'peru', 'saddle'),item.i_units IN ('Each', 'Gram', 'Oz', 'Pallet'),OR[AND[item.i_color IN ('blue', 'chartreuse'),item.i_units IN ('Each', 'Oz')],AND[item.i_color IN ('peru', 'saddle'),item.i_units IN ('Gram', 'Pallet')]]]]],AND[(item.i_category = 'Women'),item.i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[item.i_size IN ('economy', 'medium'),item.i_color IN ('aquamarine', 'blanched', 'gainsboro', 'tomato'),item.i_units IN ('Case', 'Dozen', 'Ounce', 'Tbl'),OR[AND[item.i_color IN ('aquamarine', 'gainsboro'),item.i_units IN ('Dozen', 'Ounce')],AND[item.i_color IN ('blanched', 'tomato'),item.i_units IN ('Case', 'Tbl')]]],AND[item.i_size IN ('extra large', 'small'),item.i_color IN ('almond', 'chiffon', 'lime', 'violet'),item.i_units IN ('Box', 'Dram', 'Pound', 'Ton'),OR[AND[item.i_color IN ('chiffon', 'violet'),item.i_units IN ('Pound', 'Ton')],AND[item.i_color IN ('almond', 'lime'),item.i_units IN ('Box', 'Dram')]]]]]] and item.i_category IN ('Men', 'Women')) --------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query44.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query44.out index b75453ce87346c..f1f1d1828a9985 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query44.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query44.out @@ -40,7 +40,7 @@ PhysicalResultSink ----------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------hashAgg[LOCAL] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL) +----------------------------------------------------------filter((store_sales.ss_store_sk = 146) and store_sales.ss_addr_sk IS NULL) ------------------------------------------------------------PhysicalOlapScan[store_sales] ------------------------PhysicalProject --------------------------filter((V21.rnk < 11)) @@ -66,6 +66,6 @@ PhysicalResultSink ----------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------hashAgg[LOCAL] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL) +----------------------------------------------------------filter((store_sales.ss_store_sk = 146) and store_sales.ss_addr_sk IS NULL) ------------------------------------------------------------PhysicalOlapScan[store_sales] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query45.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query45.out index 71760efa5f00fa..df39efb607722a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query45.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ws_sold_date_sk @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query46.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query46.out index 0f984367ec7ddf..ba82388fbb39bc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query46.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query46.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) +----------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() ----------------PhysicalProject @@ -25,10 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] ------------------------------------PhysicalProject ---------------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001)) +--------------------------------------filter(date_dim.d_dow IN (0, 6) and date_dim.d_year IN (1999, 2000, 2001)) ----------------------------------------PhysicalOlapScan[date_dim] --------------------------------PhysicalProject -----------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove')) +----------------------------------filter(store.s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove')) ------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)]) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query47.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query47.out index 8f2c1acd532819..f4a7774ed9115e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query47.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query47.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject -------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002)) +------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and date_dim.d_year IN (2000, 2001, 2002)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject --------------------------PhysicalOlapScan[store] @@ -38,6 +38,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1)) and (v1.s_company_name = v1_lag.s_company_name) and (v1.s_store_name = v1_lag.s_store_name)) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 s_store_name->s_store_name;RF6 s_company_name->s_company_name;RF7 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF7 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query48.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query48.out index df7d1c938fabd9..756f0b654f669a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query48.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query48.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->ss_sold_date_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->ss_addr_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[customer_address.ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[customer_address.ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->ss_addr_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF1 cd_demo_sk->ss_cdemo_sk --------------------PhysicalProject @@ -18,10 +18,10 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[store] --------------------PhysicalProject -----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W')) +----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'College', 'Primary') and customer_demographics.cd_marital_status IN ('D', 'U', 'W')) ------------------------PhysicalOlapScan[customer_demographics] ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) --------------------PhysicalOlapScan[customer_address] ------------PhysicalProject --------------filter((date_dim.d_year = 1999)) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query53.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query53.out index 996f97386c6854..065dcf28670799 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query53.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query53.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalProject @@ -20,10 +20,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------------------PhysicalProject -------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) --------------------------------------PhysicalOlapScan[item] ------------------------------PhysicalProject ---------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)) +--------------------------------filter(date_dim.d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query56.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query56.out index 68356ff9338125..e1b67bdf5440f1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query56.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((date_dim.d_moy = 2) and (date_dim.d_year = 2000)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((date_dim.d_moy = 2) and (date_dim.d_year = 2000)) @@ -75,7 +75,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((date_dim.d_moy = 2) and (date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query57.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query57.out index 6d136f83e531a4..26e56238b1094e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query57.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query57.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject -------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000)) +------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject --------------------------PhysicalOlapScan[call_center] @@ -38,6 +38,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.cc_name = v1_lag.cc_name) and (v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1))) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 cc_name->cc_name;RF6 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query58.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query58.out index c48d7e55ff0f42..9bd8eb88cb5e9d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query58.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query58.out @@ -6,9 +6,9 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id;RF14 item_id->i_item_id +------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id;RF14 item_id->i_item_id --------------PhysicalProject -----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF12 item_id->i_item_id +----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF12 item_id->i_item_id ------------------PhysicalProject --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query6.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query6.out index 8c583a1501f95f..a72581858970ea 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query6.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query6.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_month_seq = date_dim.d_month_seq)) otherCondition=() build RFs:RF4 d_month_seq->d_month_seq --------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query63.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query63.out index 04901458601acb..89ca310891b92c 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query63.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query63.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalProject @@ -20,10 +20,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------------------PhysicalProject -------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) --------------------------------------PhysicalOlapScan[item] ------------------------------PhysicalProject ---------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192)) +--------------------------------filter(date_dim.d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192)) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query64.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query64.out index 3075eaebbba62a..d0c5b61ef270aa 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query64.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query64.out @@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd1.cd_marital_status = cd2.cd_marital_status))) ------------------------------------------------PhysicalProject --------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() ----------------------------------------------------PhysicalProject @@ -62,7 +62,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------------------------------------PhysicalProject ------------------------------------------------------------------PhysicalOlapScan[date_dim(d3)] ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------filter(d_year IN (2001, 2002)) +--------------------------------------------------------------filter(d1.d_year IN (2001, 2002)) ----------------------------------------------------------------PhysicalOlapScan[date_dim(d1)] --------------------------------------------------------PhysicalProject ----------------------------------------------------------PhysicalOlapScan[store] @@ -85,7 +85,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------PhysicalProject ----------------------PhysicalOlapScan[income_band(ib2)] ----------------PhysicalProject -------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) +------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and item.i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) --------------------PhysicalOlapScan[item] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query65.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query65.out index 93cfc763ad6c44..dfefe721ea8e85 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query65.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query65.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN colocated] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk +--------------hashJoin[INNER_JOIN colocated] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(sc.revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = sc.ss_item_sk)) otherCondition=() --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query66.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query66.out index b584b6ae9834fb..2631c095ea70fd 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query66.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query66.out @@ -32,7 +32,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_time <= 77621) and (time_dim.t_time >= 48821)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter(sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) +--------------------------------filter(ship_mode.sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) ----------------------------------PhysicalOlapScan[ship_mode] --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] @@ -56,6 +56,6 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_time <= 77621) and (time_dim.t_time >= 48821)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter(sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) +--------------------------------filter(ship_mode.sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) ----------------------------------PhysicalOlapScan[ship_mode] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query68.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query68.out index d16b2c0fce83a0..0f2efbdbbf9e53 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query68.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) ----------------PhysicalProject ------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() --------------------PhysicalProject @@ -27,10 +27,10 @@ PhysicalResultSink --------------------------------------------PhysicalProject ----------------------------------------------PhysicalOlapScan[customer_address] ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill')) +--------------------------------------filter(store.s_city IN ('Five Points', 'Pleasant Hill')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)]) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query69.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query69.out index bf84c4fadb2f41..1f262518bbeab4 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query69.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query69.out @@ -35,7 +35,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer_demographics] ------------------------PhysicalProject ---------------------------filter(ca_state IN ('MI', 'TX', 'VA')) +--------------------------filter(ca.ca_state IN ('MI', 'TX', 'VA')) ----------------------------PhysicalOlapScan[customer_address(ca)] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->cs_sold_date_sk diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query71.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query71.out index e94bf6a5afa558..8de16fc964cad5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query71.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query71.out @@ -31,6 +31,6 @@ PhysicalResultSink --------------------------filter((item.i_manager_id = 1)) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(t_meal_time IN ('breakfast', 'dinner')) +----------------------filter(time_dim.t_meal_time IN ('breakfast', 'dinner')) ------------------------PhysicalOlapScan[time_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query72.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query72.out index e59874197ebd9e..39a28cec0db0e0 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query72.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query72.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk) and (d1.d_week_seq = d2.d_week_seq)) otherCondition=((cast(d_date as DATETIMEV2(0)) > cast((cast(d_date as BIGINT) + 5) as DATETIMEV2(0)))) build RFs:RF8 d_week_seq->d_week_seq;RF9 d_date_sk->cs_sold_date_sk +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk) and (d1.d_week_seq = d2.d_week_seq)) otherCondition=((cast(d3.d_date as DATETIMEV2(0)) > cast((cast(d1.d_date as BIGINT) + 5) as DATETIMEV2(0)))) build RFs:RF8 d_week_seq->d_week_seq;RF9 d_date_sk->cs_sold_date_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF7 hd_demo_sk->cs_bill_hdemo_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query73.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query73.out index b0375a6296a944..39a73327cf18e5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query73.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query73.out @@ -19,13 +19,13 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Fairfield County', 'Walker County')) +------------------------------filter(store.s_county IN ('Barrow County', 'Daviess County', 'Fairfield County', 'Walker County')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('501-1000', 'Unknown')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('501-1000', 'Unknown')) ----------------------------PhysicalOlapScan[household_demographics] ------------PhysicalProject --------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query74.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query74.out index 475085d7318237..d51aa8a437a952 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query74.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query74.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.0), (year_total / year_total), NULL) > if((year_total > 0.0), (year_total / year_total), NULL))) build RFs:RF11 customer_id->c_customer_id;RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.0), (t_w_secyear.year_total / t_w_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.0), (t_s_secyear.year_total / t_s_firstyear.year_total), NULL))) build RFs:RF11 customer_id->c_customer_id;RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF9 customer_id->c_customer_id;RF10 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query75.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query75.out index 935a49bb32d5ac..fac21f3a782ea7 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query75.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query75.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------filter((item.i_category = 'Home')) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(d_year IN (1998, 1999)) +----------------------filter(date_dim.d_year IN (1998, 1999)) ------------------------PhysicalOlapScan[date_dim] --------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------PhysicalProject @@ -39,7 +39,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------filter((item.i_category = 'Home')) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(d_year IN (1998, 1999)) +----------------------filter(date_dim.d_year IN (1998, 1999)) ------------------------PhysicalOlapScan[date_dim] --------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------PhysicalProject @@ -56,14 +56,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------filter((item.i_category = 'Home')) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(d_year IN (1998, 1999)) +----------------------filter(date_dim.d_year IN (1998, 1999)) ------------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF6 i_brand_id->i_brand_id;RF7 i_class_id->i_class_id;RF8 i_category_id->i_category_id;RF9 i_manufact_id->i_manufact_id +------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(curr_yr.sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(prev_yr.sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF6 i_brand_id->i_brand_id;RF7 i_class_id->i_class_id;RF8 i_category_id->i_category_id;RF9 i_manufact_id->i_manufact_id --------------filter((curr_yr.d_year = 1999)) ----------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 RF7 RF8 RF9 --------------filter((prev_yr.d_year = 1998)) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query76.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query76.out index 997c07fd51be53..9342e1c9fc9b1a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query76.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query76.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() --------------------------PhysicalProject -----------------------------filter(ss_hdemo_sk IS NULL) +----------------------------filter(store_sales.ss_hdemo_sk IS NULL) ------------------------------PhysicalOlapScan[store_sales] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -22,7 +22,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() --------------------------PhysicalProject -----------------------------filter(ws_bill_addr_sk IS NULL) +----------------------------filter(web_sales.ws_bill_addr_sk IS NULL) ------------------------------PhysicalOlapScan[web_sales] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -30,7 +30,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() --------------------------PhysicalProject -----------------------------filter(cs_warehouse_sk IS NULL) +----------------------------filter(catalog_sales.cs_warehouse_sk IS NULL) ------------------------------PhysicalOlapScan[catalog_sales] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query78.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query78.out index 9d12b1032063f2..3ef3609f1d0a6e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query78.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query78.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)]) +----------filter(OR[(coalesce(ws.ws_qty, 0) > 0),(coalesce(cs.cs_qty, 0) > 0)]) ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=() --------------PhysicalProject ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=() diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query79.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query79.out index bb0302a4876e86..c299cf4930bfa5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query79.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query79.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dow = 1) and d_year IN (1998, 1999, 2000)) +----------------------------------filter((date_dim.d_dow = 1) and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200)) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query8.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query8.out index 1b33582ee2b9fd..27555258c8d06a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query8.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query8.out @@ -34,10 +34,10 @@ PhysicalResultSink ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +----------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ------------------------------------------PhysicalOlapScan[customer_address] ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject ---------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +--------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ----------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query81.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query81.out index 7349d04175b27d..01529aaee2099b 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query81.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query81.out @@ -26,7 +26,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalProject ----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF4 ca_address_sk->c_current_addr_sk ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) +--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query82.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query82.out index 6dbe8967caa7b6..3182691ef4c90e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query82.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query82.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639)) +----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and item.i_manufact_id IN (138, 169, 339, 639)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09')) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query83.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query83.out index 15487151788953..8b4581b9022add 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query83.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query83.out @@ -30,7 +30,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF8 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +----------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ------------------------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------hashAgg[GLOBAL] @@ -53,7 +53,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF4 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +----------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ------------------------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashAgg[GLOBAL] @@ -76,6 +76,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) --------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query85.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query85.out index db8e373d77341c..58828ed5e85139 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query85.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query85.out @@ -13,7 +13,7 @@ PhysicalResultSink --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->ws_sold_date_sk ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->wr_refunded_addr_sk +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[customer_address.ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->wr_refunded_addr_sk ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->wr_returning_cdemo_sk;RF5 cd_marital_status->cd_marital_status;RF6 cd_education_status->cd_education_status --------------------------------PhysicalProject @@ -30,13 +30,13 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[web_page] ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W')) +--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd1.cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd1.cd_marital_status IN ('M', 'S', 'W')) ----------------------------------------PhysicalOlapScan[customer_demographics(cd1)] apply RFs: RF5 RF6 --------------------------------PhysicalProject -----------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W')) +----------------------------------filter(cd2.cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd2.cd_marital_status IN ('M', 'S', 'W')) ------------------------------------PhysicalOlapScan[customer_demographics(cd2)] ----------------------------PhysicalProject -------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) +------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) --------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------filter((date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query88.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query88.out index bfdafd7530b021..70d0a30b8fd0d6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query88.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query88.out @@ -20,7 +20,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23 ----------------------------------PhysicalProject -------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------------------PhysicalOlapScan[household_demographics] ------------------------------PhysicalProject --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30)) @@ -40,7 +40,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20 ----------------------------------PhysicalProject -------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------------------PhysicalOlapScan[household_demographics] ------------------------------PhysicalProject --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30)) @@ -60,7 +60,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17 --------------------------------PhysicalProject -----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ------------------------------------PhysicalOlapScan[household_demographics] ----------------------------PhysicalProject ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30)) @@ -80,7 +80,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14 ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30)) @@ -100,7 +100,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11 ----------------------------PhysicalProject -------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30)) @@ -120,7 +120,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8 --------------------------PhysicalProject -----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ------------------------------PhysicalOlapScan[household_demographics] ----------------------PhysicalProject ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30)) @@ -140,7 +140,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5 ------------------------PhysicalProject ---------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30)) @@ -160,7 +160,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------PhysicalProject -------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------PhysicalOlapScan[household_demographics] ------------------PhysicalProject --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30)) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query89.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query89.out index 45741d4ce87714..21d97fe65a667d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query89.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query89.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (avg_monthly_sales = 0.0000)) and ((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) +------------filter(( not (tmp1.avg_monthly_sales = 0.0000)) and ((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------hashAgg[GLOBAL] @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious')) +--------------------------------------filter(OR[AND[item.i_category IN ('Electronics', 'Jewelry', 'Shoes'),item.i_class IN ('athletic', 'portable', 'semi-precious')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'maternity', 'rock')]] and item.i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and item.i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 1999)) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query91.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query91.out index 80fb9d20ee16fd..73262c770a1826 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query91.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query91.out @@ -33,9 +33,9 @@ PhysicalResultSink ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) --------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter((hd_buy_potential like '1001-5000%')) +----------------------filter((household_demographics.hd_buy_potential like '1001-5000%')) ------------------------PhysicalOlapScan[household_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query92.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query92.out index b02739c84db1ba..b69e7b837d186d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query92.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query92.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +------------filter((cast(web_sales.ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query94.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query94.out index 3344008e03754a..1abd7ec2887d54 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query94.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query94.out @@ -16,7 +16,7 @@ PhysicalResultSink --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ws_ship_date_sk ----------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=() ------------------------------PhysicalProject ---------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +--------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF0 ----------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query95.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query95.out index 3a5cc0e6006683..0e0514695e2a75 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query95.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF8 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query98.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query98.out index b9f4c136c030a7..9bd80eb5992ac1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query98.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query98.out @@ -17,7 +17,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------------PhysicalProject -------------------------------filter(i_category IN ('Music', 'Shoes', 'Sports')) +------------------------------filter(item.i_category IN ('Music', 'Shoes', 'Sports')) --------------------------------PhysicalOlapScan[item] ------------------------PhysicalProject --------------------------filter((date_dim.d_date <= '2002-06-19') and (date_dim.d_date >= '2002-05-20')) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query1.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query1.out index 95157b263f4a4e..760a5bbd19e3a3 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query1.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query1.out @@ -18,7 +18,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF3 ctr_store_sk->ctr_store_sk;RF4 ctr_store_sk->s_store_sk +------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF3 ctr_store_sk->ctr_store_sk;RF4 ctr_store_sk->s_store_sk --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->ctr_store_sk ------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query10.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query10.out index f11e93d1dc045e..7eec4d7c91c324 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query10.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query10.out @@ -35,7 +35,7 @@ PhysicalResultSink ----------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalOlapScan[customer_demographics] --------------------------PhysicalProject -----------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County')) +----------------------------filter(ca.ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County')) ------------------------------PhysicalOlapScan[customer_address(ca)] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->cs_sold_date_sk diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query11.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query11.out index a3f5a7e79a7c7c..d66200f1600cb5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query11.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query11.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000) > if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000))) build RFs:RF11 customer_id->c_customer_id;RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(38, 8)) / t_w_firstyear.year_total), 0.000000) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(38, 8)) / t_s_firstyear.year_total), 0.000000))) build RFs:RF11 customer_id->c_customer_id;RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF9 customer_id->c_customer_id;RF10 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query12.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query12.out index dcf5a48b347927..3fcbb69e60cbef 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query12.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query12.out @@ -17,7 +17,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 ----------------------------PhysicalProject -------------------------------filter(i_category IN ('Books', 'Men', 'Sports')) +------------------------------filter(item.i_category IN ('Books', 'Men', 'Sports')) --------------------------------PhysicalOlapScan[item] ------------------------PhysicalProject --------------------------filter((date_dim.d_date <= '1998-05-06') and (date_dim.d_date >= '1998-04-06')) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query13.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query13.out index b922267cfd2004..bb8bc5d57189c9 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query13.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query13.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->ss_sold_date_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->ss_addr_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->ss_addr_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->ss_hdemo_sk --------------------PhysicalProject @@ -20,13 +20,13 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')]] and customer_demographics.cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and customer_demographics.cd_marital_status IN ('D', 'M', 'S')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter(hd_dep_count IN (1, 3)) +----------------------filter(household_demographics.hd_dep_count IN (1, 3)) ------------------------PhysicalOlapScan[household_demographics] ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'KS', 'MI', 'MO', 'ND', 'NH', 'OH', 'SD', 'TX')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('CO', 'KS', 'MI', 'MO', 'ND', 'NH', 'OH', 'SD', 'TX')) --------------------PhysicalOlapScan[customer_address] ------------PhysicalProject --------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query15.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query15.out index 82a4dc7105b37f..4411928af9d355 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query15.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query15.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF1 ca_address_sk->c_current_addr_sk +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),customer_address.ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF1 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF0 c_customer_sk->cs_bill_customer_sk --------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query16.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query16.out index a77fc751be12e3..c5ddb58c3b1730 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query16.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query16.out @@ -16,7 +16,7 @@ PhysicalResultSink --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_ship_date_sk ----------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=() ------------------------------PhysicalProject ---------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF0 cs_order_number->cs_order_number +--------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF0 cs_order_number->cs_order_number ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF0 ----------------------------------PhysicalProject @@ -30,6 +30,6 @@ PhysicalResultSink --------------------------filter((customer_address.ca_state = 'WV')) ----------------------------PhysicalOlapScan[customer_address] --------------------PhysicalProject -----------------------filter(cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County')) +----------------------filter(call_center.cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County')) ------------------------PhysicalOlapScan[call_center] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query17.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query17.out index fbdb6bff1cd4b7..2cac188fe579e8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query17.out @@ -34,10 +34,10 @@ PhysicalResultSink ----------------------------------filter((d1.d_quarter_name = '2001Q1')) ------------------------------------PhysicalOlapScan[date_dim(d1)] ----------------------------PhysicalProject -------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +------------------------------filter(d2.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) --------------------------------PhysicalOlapScan[date_dim(d2)] ------------------------PhysicalProject ---------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------filter(d3.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------PhysicalOlapScan[date_dim(d3)] --------------------PhysicalProject ----------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query18.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query18.out index 0b11396262bcde..dee6ccfd762094 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query18.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query18.out @@ -27,12 +27,12 @@ PhysicalResultSink --------------------------------------------filter((cd1.cd_education_status = 'Advanced Degree') and (cd1.cd_gender = 'F')) ----------------------------------------------PhysicalOlapScan[customer_demographics(cd1)] --------------------------------------PhysicalProject -----------------------------------------filter(c_birth_month IN (1, 10, 2, 4, 7, 8)) +----------------------------------------filter(customer.c_birth_month IN (1, 10, 2, 4, 7, 8)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF2 RF3 ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer_demographics(cd2)] ------------------------------PhysicalProject ---------------------------------filter(ca_state IN ('GA', 'IN', 'ME', 'NC', 'OK', 'WA', 'WY')) +--------------------------------filter(customer_address.ca_state IN ('GA', 'IN', 'ME', 'NC', 'OK', 'WA', 'WY')) ----------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query19.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query19.out index e736856716d608..27ae851393f863 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query19.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ss_item_sk ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query20.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query20.out index 8fc1e09860c453..9a59577c26c34e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query20.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query20.out @@ -17,7 +17,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 ----------------------------PhysicalProject -------------------------------filter(i_category IN ('Books', 'Shoes', 'Women')) +------------------------------filter(item.i_category IN ('Books', 'Shoes', 'Women')) --------------------------------PhysicalOlapScan[item] ------------------------PhysicalProject --------------------------filter((date_dim.d_date <= '2002-02-25') and (date_dim.d_date >= '2002-01-26')) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query21.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query21.out index 579e0d5ae9421c..911edd2fa1cee5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query21.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query21.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) <= 1.5) and (if((inv_before > 0), (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) +--------filter(((cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)) <= 1.5) and (if((x.inv_before > 0), (cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query23.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query23.out index d8fbe3f3c1d24d..43dfa5ec0bd495 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query23.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query23.out @@ -16,7 +16,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[item] ------------------PhysicalProject ---------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------PhysicalOlapScan[date_dim] --PhysicalCteAnchor ( cteId=CTEId#2 ) ----PhysicalCteProducer ( cteId=CTEId#2 ) @@ -27,7 +27,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------filter(( not ss_customer_sk IS NULL)) +--------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------PhysicalOlapScan[store_sales] ----------PhysicalProject ------------hashAgg[GLOBAL] @@ -40,10 +40,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ------------------------------PhysicalProject ---------------------------------filter(( not ss_customer_sk IS NULL)) +--------------------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ------------------------------PhysicalProject ---------------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query24.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query24.out index 8a985dbfbec73d..8a5292b3c8912f 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query24.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query24.out @@ -9,7 +9,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF6 s_zip->ca_zip;RF7 s_store_sk->ss_store_sk ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF5 ca_address_sk->c_current_addr_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (customer.c_birth_country = upper(customer_address.ca_country)))) build RFs:RF5 ca_address_sk->c_current_addr_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->ss_customer_sk ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query27.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query27.out index 1fe1f20ca9ce98..545b9b37091563 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query27.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query27.out @@ -28,6 +28,6 @@ PhysicalResultSink ----------------------------filter((date_dim.d_year = 1999)) ------------------------------PhysicalOlapScan[date_dim] ----------------------PhysicalProject -------------------------filter(s_state IN ('AL', 'LA', 'MI', 'MO', 'SC', 'TN')) +------------------------filter(store.s_state IN ('AL', 'LA', 'MI', 'MO', 'SC', 'TN')) --------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query29.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query29.out index 72706186c5e584..83afa2029ccc95 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query29.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query29.out @@ -36,7 +36,7 @@ PhysicalResultSink ----------------------------filter((d2.d_moy <= 7) and (d2.d_moy >= 4) and (d2.d_year = 1999)) ------------------------------PhysicalOlapScan[date_dim(d2)] ----------------------PhysicalProject -------------------------filter(d_year IN (1999, 2000, 2001)) +------------------------filter(d3.d_year IN (1999, 2000, 2001)) --------------------------PhysicalOlapScan[date_dim(d3)] ------------------PhysicalProject --------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query30.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query30.out index d5811d89bfecca..37fa39ba887a0e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query30.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query30.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query31.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query31.out index c8599630d82557..f96ed8fce5691a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query31.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query31.out @@ -15,7 +15,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject -------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +------------------filter((ss.d_year = 2000) and ss.d_qoy IN (1, 2, 3)) --------------------PhysicalOlapScan[date_dim] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) @@ -32,16 +32,16 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer_address] ------------------PhysicalProject ---------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) +--------------------filter((ws.d_year = 2000) and ws.d_qoy IN (1, 2, 3)) ----------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalQuickSort[MERGE_SORT] --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalQuickSort[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF13 ca_county->ca_county;RF14 ca_county->ca_county;RF15 ca_county->ca_county;RF16 ca_county->ca_county +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((ws2.web_sales > 0.00), (cast(ws3.web_sales as DECIMALV3(38, 8)) / ws2.web_sales), NULL) > if((ss2.store_sales > 0.00), (cast(ss3.store_sales as DECIMALV3(38, 8)) / ss2.store_sales), NULL))) build RFs:RF13 ca_county->ca_county;RF14 ca_county->ca_county;RF15 ca_county->ca_county;RF16 ca_county->ca_county ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF9 ca_county->ca_county;RF10 ca_county->ca_county;RF11 ca_county->ca_county +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((ws1.web_sales > 0.00), (cast(ws2.web_sales as DECIMALV3(38, 8)) / ws1.web_sales), NULL) > if((ss1.store_sales > 0.00), (cast(ss2.store_sales as DECIMALV3(38, 8)) / ss1.store_sales), NULL))) build RFs:RF9 ca_county->ca_county;RF10 ca_county->ca_county;RF11 ca_county->ca_county --------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss1.ca_county = ws1.ca_county)) otherCondition=() build RFs:RF6 ca_county->ca_county;RF7 ca_county->ca_county;RF12 ca_county->ca_county;RF17 ca_county->ca_county ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ss2.ca_county = ss3.ca_county)) otherCondition=() build RFs:RF5 ca_county->ca_county diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query32.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query32.out index ef8807bfdbf3ef..e252239b8dfac2 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query32.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query32.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +--------------filter((cast(catalog_sales.cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) ----------------PhysicalWindow ------------------PhysicalQuickSort[LOCAL_SORT] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query34.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query34.out index af70b27d23d2fa..6266f17bfeedf1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query34.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query34.out @@ -19,13 +19,13 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000)) +----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County')) +------------------------------filter(store.s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('0-500', '1001-5000')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('0-500', '1001-5000')) ----------------------------PhysicalOlapScan[household_demographics] ------------PhysicalProject --------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query36.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query36.out index 9c088300d0c51b..1c9b225ddaf401 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query36.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query36.out @@ -28,6 +28,6 @@ PhysicalResultSink --------------------------------------filter((d1.d_year = 2002)) ----------------------------------------PhysicalOlapScan[date_dim(d1)] --------------------------------PhysicalProject -----------------------------------filter(s_state IN ('AL', 'GA', 'MI', 'MO', 'OH', 'SC', 'SD', 'TN')) +----------------------------------filter(store.s_state IN ('AL', 'GA', 'MI', 'MO', 'OH', 'SC', 'SD', 'TN')) ------------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query37.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query37.out index e2448337a82e48..7122ceb72487d6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query37.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query37.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856)) +----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and item.i_manufact_id IN (1000, 707, 747, 856)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21')) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query39.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query39.out index 0c9b71789f9ae6..97ec51639d3f46 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query39.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter(( not (mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) +------filter(( not (foo.mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) --------hashAgg[GLOBAL] ----------PhysicalProject ------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->inv_date_sk @@ -17,7 +17,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------PhysicalProject --------------------PhysicalOlapScan[warehouse] --------------PhysicalProject -----------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2)) +----------------filter((date_dim.d_year = 1998) and inv.d_moy IN (1, 2)) ------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query4.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query4.out index 479aef2c34a707..41ffa4481da213 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query4.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query4.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_w_firstyear.year_total > 0.000000), (cast(t_w_secyear.year_total as DECIMALV3(38, 16)) / t_w_firstyear.year_total), NULL))) build RFs:RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF18 customer_id->c_customer_id;RF19 customer_id->c_customer_id;RF20 customer_id->c_customer_id;RF21 customer_id->c_customer_id ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF15 customer_id->c_customer_id;RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.000000), (cast(t_s_secyear.year_total as DECIMALV3(38, 16)) / t_s_firstyear.year_total), NULL))) build RFs:RF15 customer_id->c_customer_id;RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF12 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query41.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query41.out index c7c720dd75259e..12dad98df5b0fc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query41.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query41.out @@ -18,6 +18,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'medium'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('dodger', 'indian', 'spring', 'tan'),i_units IN ('Bunch', 'Carton', 'Tsp', 'Unknown'),OR[AND[i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp')],AND[i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown')]]],AND[i_size IN ('N/A', 'large'),i_color IN ('blue', 'chartreuse', 'peru', 'saddle'),i_units IN ('Each', 'Gram', 'Oz', 'Pallet'),OR[AND[i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz')],AND[i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet')]]]]],AND[(item.i_category = 'Women'),i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('aquamarine', 'blanched', 'gainsboro', 'tomato'),i_units IN ('Case', 'Dozen', 'Ounce', 'Tbl'),OR[AND[i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce')],AND[i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl')]]],AND[i_size IN ('extra large', 'small'),i_color IN ('almond', 'chiffon', 'lime', 'violet'),i_units IN ('Box', 'Dram', 'Pound', 'Ton'),OR[AND[i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton')],AND[i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram')]]]]]] and i_category IN ('Men', 'Women')) +------------------------------filter(OR[AND[(item.i_category = 'Men'),item.i_size IN ('N/A', 'economy', 'large', 'medium'),OR[AND[item.i_size IN ('economy', 'medium'),item.i_color IN ('dodger', 'indian', 'spring', 'tan'),item.i_units IN ('Bunch', 'Carton', 'Tsp', 'Unknown'),OR[AND[item.i_color IN ('dodger', 'tan'),item.i_units IN ('Bunch', 'Tsp')],AND[item.i_color IN ('indian', 'spring'),item.i_units IN ('Carton', 'Unknown')]]],AND[item.i_size IN ('N/A', 'large'),item.i_color IN ('blue', 'chartreuse', 'peru', 'saddle'),item.i_units IN ('Each', 'Gram', 'Oz', 'Pallet'),OR[AND[item.i_color IN ('blue', 'chartreuse'),item.i_units IN ('Each', 'Oz')],AND[item.i_color IN ('peru', 'saddle'),item.i_units IN ('Gram', 'Pallet')]]]]],AND[(item.i_category = 'Women'),item.i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[item.i_size IN ('economy', 'medium'),item.i_color IN ('aquamarine', 'blanched', 'gainsboro', 'tomato'),item.i_units IN ('Case', 'Dozen', 'Ounce', 'Tbl'),OR[AND[item.i_color IN ('aquamarine', 'gainsboro'),item.i_units IN ('Dozen', 'Ounce')],AND[item.i_color IN ('blanched', 'tomato'),item.i_units IN ('Case', 'Tbl')]]],AND[item.i_size IN ('extra large', 'small'),item.i_color IN ('almond', 'chiffon', 'lime', 'violet'),item.i_units IN ('Box', 'Dram', 'Pound', 'Ton'),OR[AND[item.i_color IN ('chiffon', 'violet'),item.i_units IN ('Pound', 'Ton')],AND[item.i_color IN ('almond', 'lime'),item.i_units IN ('Box', 'Dram')]]]]]] and item.i_category IN ('Men', 'Women')) --------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query44.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query44.out index b75453ce87346c..f1f1d1828a9985 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query44.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query44.out @@ -40,7 +40,7 @@ PhysicalResultSink ----------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------hashAgg[LOCAL] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL) +----------------------------------------------------------filter((store_sales.ss_store_sk = 146) and store_sales.ss_addr_sk IS NULL) ------------------------------------------------------------PhysicalOlapScan[store_sales] ------------------------PhysicalProject --------------------------filter((V21.rnk < 11)) @@ -66,6 +66,6 @@ PhysicalResultSink ----------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------hashAgg[LOCAL] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL) +----------------------------------------------------------filter((store_sales.ss_store_sk = 146) and store_sales.ss_addr_sk IS NULL) ------------------------------------------------------------PhysicalOlapScan[store_sales] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query45.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query45.out index d334ffefada826..020664647969c8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query45.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ws_sold_date_sk @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query46.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query46.out index f4ec4d7453375e..3c897485dde572 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query46.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query46.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk +----------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->ss_customer_sk ----------------PhysicalProject @@ -25,10 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer_address] ------------------------------------PhysicalProject ---------------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001)) +--------------------------------------filter(date_dim.d_dow IN (0, 6) and date_dim.d_year IN (1999, 2000, 2001)) ----------------------------------------PhysicalOlapScan[date_dim] --------------------------------PhysicalProject -----------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove')) +----------------------------------filter(store.s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove')) ------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)]) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query47.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query47.out index 76597c14196cba..9302070ffa158d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query47.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query47.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject -------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002)) +------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and date_dim.d_year IN (2000, 2001, 2002)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject --------------------------PhysicalOlapScan[store] @@ -38,6 +38,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1)) and (v1.s_company_name = v1_lag.s_company_name) and (v1.s_store_name = v1_lag.s_store_name)) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 s_store_name->s_store_name;RF6 s_company_name->s_company_name;RF7 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF7 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query48.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query48.out index 16b26116f8fc05..d0c9b3bd819813 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query48.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query48.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->ss_sold_date_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->ss_addr_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[customer_address.ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[customer_address.ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->ss_addr_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF1 cd_demo_sk->ss_cdemo_sk --------------------PhysicalProject @@ -18,10 +18,10 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[store] --------------------PhysicalProject -----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W')) +----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'College', 'Primary') and customer_demographics.cd_marital_status IN ('D', 'U', 'W')) ------------------------PhysicalOlapScan[customer_demographics] ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) --------------------PhysicalOlapScan[customer_address] ------------PhysicalProject --------------filter((date_dim.d_year = 1999)) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query53.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query53.out index c8d20c2a71fa70..6f5cd32cdf58e4 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query53.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query53.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalProject @@ -20,10 +20,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------PhysicalProject -------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) --------------------------------------PhysicalOlapScan[item] ------------------------------PhysicalProject ---------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)) +--------------------------------filter(date_dim.d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query56.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query56.out index 68356ff9338125..e1b67bdf5440f1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query56.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((date_dim.d_moy = 2) and (date_dim.d_year = 2000)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((date_dim.d_moy = 2) and (date_dim.d_year = 2000)) @@ -75,7 +75,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((date_dim.d_moy = 2) and (date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query57.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query57.out index b99368bce604a6..90cc08418d694c 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query57.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query57.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject -------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000)) +------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject --------------------------PhysicalOlapScan[call_center] @@ -38,6 +38,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.cc_name = v1_lag.cc_name) and (v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1))) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 cc_name->cc_name;RF6 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query58.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query58.out index 54c1b4bb6dc07e..5c17e07ce465b1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query58.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query58.out @@ -6,9 +6,9 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id;RF14 item_id->i_item_id +------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id;RF14 item_id->i_item_id --------------PhysicalProject -----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF12 item_id->i_item_id +----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF12 item_id->i_item_id ------------------PhysicalProject --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query6.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query6.out index bcc2a2d1ea0749..b9dedae61059c2 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query6.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query6.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF5 i_category->i_category +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF5 i_category->i_category ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_month_seq = date_dim.d_month_seq)) otherCondition=() build RFs:RF4 d_month_seq->d_month_seq --------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query63.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query63.out index 3d189c44304060..ce71384b440f60 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query63.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query63.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalProject @@ -20,10 +20,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------PhysicalProject -------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) --------------------------------------PhysicalOlapScan[item] ------------------------------PhysicalProject ---------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192)) +--------------------------------filter(date_dim.d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192)) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query64.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query64.out index 7144260c6c196c..302bea870de67d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query64.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query64.out @@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF13 p_promo_sk->ss_promo_sk --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF12 cd_demo_sk->c_current_cdemo_sk +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd1.cd_marital_status = cd2.cd_marital_status))) build RFs:RF12 cd_demo_sk->c_current_cdemo_sk ------------------------------------------------PhysicalProject --------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() build RFs:RF11 cd_demo_sk->ss_cdemo_sk ----------------------------------------------------PhysicalProject @@ -62,7 +62,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------------------------------------PhysicalProject ------------------------------------------------------------------PhysicalOlapScan[date_dim(d3)] ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------filter(d_year IN (2001, 2002)) +--------------------------------------------------------------filter(d1.d_year IN (2001, 2002)) ----------------------------------------------------------------PhysicalOlapScan[date_dim(d1)] --------------------------------------------------------PhysicalProject ----------------------------------------------------------PhysicalOlapScan[store] @@ -85,7 +85,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------PhysicalProject ----------------------PhysicalOlapScan[income_band(ib2)] ----------------PhysicalProject -------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) +------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and item.i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) --------------------PhysicalOlapScan[item] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query65.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query65.out index 3d0f8f9dc136c0..ded258d968170e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query65.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query65.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN colocated] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk +--------------hashJoin[INNER_JOIN colocated] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(sc.revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = sc.ss_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ss_item_sk --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query66.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query66.out index 8b734945f5f417..7b617b2731d98a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query66.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query66.out @@ -32,7 +32,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_time <= 77621) and (time_dim.t_time >= 48821)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter(sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) +--------------------------------filter(ship_mode.sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) ----------------------------------PhysicalOlapScan[ship_mode] --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] @@ -56,6 +56,6 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_time <= 77621) and (time_dim.t_time >= 48821)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter(sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) +--------------------------------filter(ship_mode.sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) ----------------------------------PhysicalOlapScan[ship_mode] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query68.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query68.out index 53b709f303c59e..63b5f303548d84 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query68.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->ss_customer_sk --------------------PhysicalProject @@ -27,10 +27,10 @@ PhysicalResultSink --------------------------------------------PhysicalProject ----------------------------------------------PhysicalOlapScan[customer_address] ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill')) +--------------------------------------filter(store.s_city IN ('Five Points', 'Pleasant Hill')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)]) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query69.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query69.out index dcc12f5b0d9036..5f807727944bd0 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query69.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query69.out @@ -35,7 +35,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer_demographics] ------------------------PhysicalProject ---------------------------filter(ca_state IN ('MI', 'TX', 'VA')) +--------------------------filter(ca.ca_state IN ('MI', 'TX', 'VA')) ----------------------------PhysicalOlapScan[customer_address(ca)] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->cs_sold_date_sk diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query71.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query71.out index e94bf6a5afa558..8de16fc964cad5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query71.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query71.out @@ -31,6 +31,6 @@ PhysicalResultSink --------------------------filter((item.i_manager_id = 1)) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(t_meal_time IN ('breakfast', 'dinner')) +----------------------filter(time_dim.t_meal_time IN ('breakfast', 'dinner')) ------------------------PhysicalOlapScan[time_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query72.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query72.out index 992757ceb6ca8e..270f06c722883b 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query72.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query72.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk) and (d1.d_week_seq = d2.d_week_seq)) otherCondition=((cast(d_date as DATETIMEV2(0)) > cast((cast(d_date as BIGINT) + 5) as DATETIMEV2(0)))) build RFs:RF8 d_week_seq->d_week_seq;RF9 d_date_sk->cs_sold_date_sk +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk) and (d1.d_week_seq = d2.d_week_seq)) otherCondition=((cast(d3.d_date as DATETIMEV2(0)) > cast((cast(d1.d_date as BIGINT) + 5) as DATETIMEV2(0)))) build RFs:RF8 d_week_seq->d_week_seq;RF9 d_date_sk->cs_sold_date_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF7 hd_demo_sk->cs_bill_hdemo_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query73.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query73.out index 79261c5e1d732c..da994405ae5c2e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query73.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query73.out @@ -19,13 +19,13 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Fairfield County', 'Walker County')) +------------------------------filter(store.s_county IN ('Barrow County', 'Daviess County', 'Fairfield County', 'Walker County')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('501-1000', 'Unknown')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('501-1000', 'Unknown')) ----------------------------PhysicalOlapScan[household_demographics] ------------PhysicalProject --------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query74.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query74.out index fd107814a01da7..dec461bb511a74 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query74.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query74.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.0), (year_total / year_total), NULL) > if((year_total > 0.0), (year_total / year_total), NULL))) build RFs:RF11 customer_id->c_customer_id;RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.0), (t_w_secyear.year_total / t_w_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.0), (t_s_secyear.year_total / t_s_firstyear.year_total), NULL))) build RFs:RF11 customer_id->c_customer_id;RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF9 customer_id->c_customer_id;RF10 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query75.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query75.out index 935a49bb32d5ac..fac21f3a782ea7 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query75.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query75.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------filter((item.i_category = 'Home')) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(d_year IN (1998, 1999)) +----------------------filter(date_dim.d_year IN (1998, 1999)) ------------------------PhysicalOlapScan[date_dim] --------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------PhysicalProject @@ -39,7 +39,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------filter((item.i_category = 'Home')) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(d_year IN (1998, 1999)) +----------------------filter(date_dim.d_year IN (1998, 1999)) ------------------------PhysicalOlapScan[date_dim] --------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------PhysicalProject @@ -56,14 +56,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------filter((item.i_category = 'Home')) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(d_year IN (1998, 1999)) +----------------------filter(date_dim.d_year IN (1998, 1999)) ------------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF6 i_brand_id->i_brand_id;RF7 i_class_id->i_class_id;RF8 i_category_id->i_category_id;RF9 i_manufact_id->i_manufact_id +------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(curr_yr.sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(prev_yr.sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF6 i_brand_id->i_brand_id;RF7 i_class_id->i_class_id;RF8 i_category_id->i_category_id;RF9 i_manufact_id->i_manufact_id --------------filter((curr_yr.d_year = 1999)) ----------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 RF7 RF8 RF9 --------------filter((prev_yr.d_year = 1998)) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query76.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query76.out index 4c13d64bbaeee4..dfa4e054cb4da1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query76.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query76.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->ss_item_sk --------------------------PhysicalProject -----------------------------filter(ss_hdemo_sk IS NULL) +----------------------------filter(store_sales.ss_hdemo_sk IS NULL) ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF3 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -22,7 +22,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->ws_item_sk --------------------------PhysicalProject -----------------------------filter(ws_bill_addr_sk IS NULL) +----------------------------filter(web_sales.ws_bill_addr_sk IS NULL) ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF4 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -30,7 +30,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk --------------------------PhysicalProject -----------------------------filter(cs_warehouse_sk IS NULL) +----------------------------filter(catalog_sales.cs_warehouse_sk IS NULL) ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF5 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query78.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query78.out index 9d12b1032063f2..3ef3609f1d0a6e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query78.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query78.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)]) +----------filter(OR[(coalesce(ws.ws_qty, 0) > 0),(coalesce(cs.cs_qty, 0) > 0)]) ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=() --------------PhysicalProject ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=() diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query79.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query79.out index 756f1ddad3673e..55bd6d8bda41a3 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query79.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query79.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dow = 1) and d_year IN (1998, 1999, 2000)) +----------------------------------filter((date_dim.d_dow = 1) and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200)) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query8.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query8.out index 02488c50cc2afe..9969423be02fc3 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query8.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query8.out @@ -34,10 +34,10 @@ PhysicalResultSink ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +----------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ------------------------------------------PhysicalOlapScan[customer_address] ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject ---------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +--------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ----------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query81.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query81.out index 33d83bce322733..ac9885570fa2c5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query81.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query81.out @@ -26,7 +26,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalProject ----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF4 ca_address_sk->c_current_addr_sk ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF3 ctr_state->ctr_state +--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF3 ctr_state->ctr_state ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->ctr_customer_sk --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF2 RF3 diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query82.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query82.out index 6dbe8967caa7b6..3182691ef4c90e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query82.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query82.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639)) +----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and item.i_manufact_id IN (138, 169, 339, 639)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09')) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query83.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query83.out index a942211e1d29c7..2010f134e7d058 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query83.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query83.out @@ -30,7 +30,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF8 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +----------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ------------------------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------hashAgg[GLOBAL] @@ -53,7 +53,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF4 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +----------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ------------------------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashAgg[GLOBAL] @@ -76,6 +76,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) --------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query85.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query85.out index 92df74c7ae33ba..5211dcdb5eabb5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query85.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query85.out @@ -13,7 +13,7 @@ PhysicalResultSink --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->ws_sold_date_sk ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->wr_refunded_addr_sk +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[customer_address.ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->wr_refunded_addr_sk ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->wr_returning_cdemo_sk;RF5 cd_marital_status->cd_marital_status;RF6 cd_education_status->cd_education_status --------------------------------PhysicalProject @@ -30,13 +30,13 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[web_page] ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W')) +--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd1.cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd1.cd_marital_status IN ('M', 'S', 'W')) ----------------------------------------PhysicalOlapScan[customer_demographics(cd1)] apply RFs: RF5 RF6 --------------------------------PhysicalProject -----------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W')) +----------------------------------filter(cd2.cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd2.cd_marital_status IN ('M', 'S', 'W')) ------------------------------------PhysicalOlapScan[customer_demographics(cd2)] ----------------------------PhysicalProject -------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) +------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) --------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------filter((date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query88.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query88.out index bfdafd7530b021..70d0a30b8fd0d6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query88.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query88.out @@ -20,7 +20,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23 ----------------------------------PhysicalProject -------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------------------PhysicalOlapScan[household_demographics] ------------------------------PhysicalProject --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30)) @@ -40,7 +40,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20 ----------------------------------PhysicalProject -------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------------------PhysicalOlapScan[household_demographics] ------------------------------PhysicalProject --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30)) @@ -60,7 +60,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17 --------------------------------PhysicalProject -----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ------------------------------------PhysicalOlapScan[household_demographics] ----------------------------PhysicalProject ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30)) @@ -80,7 +80,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14 ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30)) @@ -100,7 +100,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11 ----------------------------PhysicalProject -------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30)) @@ -120,7 +120,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8 --------------------------PhysicalProject -----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ------------------------------PhysicalOlapScan[household_demographics] ----------------------PhysicalProject ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30)) @@ -140,7 +140,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5 ------------------------PhysicalProject ---------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30)) @@ -160,7 +160,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------PhysicalProject -------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------PhysicalOlapScan[household_demographics] ------------------PhysicalProject --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30)) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query89.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query89.out index fecb0375185f1c..4032f2190b0d4d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query89.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query89.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (avg_monthly_sales = 0.0000)) and ((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) +------------filter(( not (tmp1.avg_monthly_sales = 0.0000)) and ((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------hashAgg[GLOBAL] @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious')) +--------------------------------------filter(OR[AND[item.i_category IN ('Electronics', 'Jewelry', 'Shoes'),item.i_class IN ('athletic', 'portable', 'semi-precious')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'maternity', 'rock')]] and item.i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and item.i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 1999)) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query91.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query91.out index 5b97ad85277e69..469ef82b138be6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query91.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query91.out @@ -33,9 +33,9 @@ PhysicalResultSink ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) --------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter((hd_buy_potential like '1001-5000%')) +----------------------filter((household_demographics.hd_buy_potential like '1001-5000%')) ------------------------PhysicalOlapScan[household_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query92.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query92.out index b02739c84db1ba..b69e7b837d186d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query92.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query92.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +------------filter((cast(web_sales.ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query94.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query94.out index 3344008e03754a..1abd7ec2887d54 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query94.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query94.out @@ -16,7 +16,7 @@ PhysicalResultSink --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ws_ship_date_sk ----------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=() ------------------------------PhysicalProject ---------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +--------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF0 ----------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query95.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query95.out index d6897723d3b21b..0476c1e198f355 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query95.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number;RF1 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number;RF1 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF1 RF16 RF18 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query98.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query98.out index b9f4c136c030a7..9bd80eb5992ac1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query98.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query98.out @@ -17,7 +17,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------------PhysicalProject -------------------------------filter(i_category IN ('Music', 'Shoes', 'Sports')) +------------------------------filter(item.i_category IN ('Music', 'Shoes', 'Sports')) --------------------------------PhysicalOlapScan[item] ------------------------PhysicalProject --------------------------filter((date_dim.d_date <= '2002-06-19') and (date_dim.d_date >= '2002-05-20')) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query1.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query1.out index 415ac836f0d9be..8f705612f3c581 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query1.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query1.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalProject ----------------PhysicalOlapScan[customer] apply RFs: RF4 --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->ctr_store_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query10.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query10.out index d9535f09438e5e..556775c934047f 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query10.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query10.out @@ -42,6 +42,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County')) +------------------------------------filter(ca.ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County')) --------------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query11.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query11.out index 42741b2cefc301..802b6eeedff2fb 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query11.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query11.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000) > if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000))) build RFs:RF11 c_customer_sk->ws_bill_customer_sk +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(38, 8)) / t_w_firstyear.year_total), 0.000000) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(38, 8)) / t_s_firstyear.year_total), 0.000000))) build RFs:RF11 c_customer_sk->ws_bill_customer_sk ------------PhysicalProject --------------hashAgg[GLOBAL] ----------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query12.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query12.out index c7d13441a9337c..12af901e79fdbd 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query12.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query12.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '1998-05-06') and (date_dim.d_date >= '1998-04-06')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Men', 'Sports')) +----------------------------filter(item.i_category IN ('Books', 'Men', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query13.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query13.out index 5b0b79f9465132..da3fcf839818a9 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query13.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query13.out @@ -9,25 +9,25 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->cd_demo_sk ----------------PhysicalProject -------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S')) +------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')]] and customer_demographics.cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and customer_demographics.cd_marital_status IN ('D', 'M', 'S')) --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->ss_hdemo_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ss_sold_date_sk ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->ss_addr_sk +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->ss_addr_sk ----------------------------PhysicalProject ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------PhysicalProject -------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'KS', 'MI', 'MO', 'ND', 'NH', 'OH', 'SD', 'TX')) +------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('CO', 'KS', 'MI', 'MO', 'ND', 'NH', 'OH', 'SD', 'TX')) --------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------filter((date_dim.d_year = 2001)) ----------------------------PhysicalOlapScan[date_dim] --------------------PhysicalProject -----------------------filter(hd_dep_count IN (1, 3)) +----------------------filter(household_demographics.hd_dep_count IN (1, 3)) ------------------------PhysicalOlapScan[household_demographics] ------------PhysicalProject --------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query15.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query15.out index e5d7debf7fec2c..c0c9d78bd02dbc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query15.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query15.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),customer_address.ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_sold_date_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query16.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query16.out index 3b148bf6fc8b24..1c4377348abffb 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query16.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query16.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF3 cs_order_number->cs_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF3 cs_order_number->cs_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF3 --------------------PhysicalProject @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------------filter((date_dim.d_date <= '2002-05-31') and (date_dim.d_date >= '2002-04-01')) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject ---------------------------filter(cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County')) +--------------------------filter(call_center.cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County')) ----------------------------PhysicalOlapScan[call_center] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query17.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query17.out index c63faa3bdb5843..9ced08d3ccfca5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query17.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query17.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF8 RF9 RF10 ------------------------PhysicalProject ---------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------filter(d3.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------PhysicalOlapScan[date_dim(d3)] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() @@ -35,7 +35,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0 ------------------------------------PhysicalProject ---------------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------------------filter(d2.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------------------PhysicalOlapScan[date_dim(d2)] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query18.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query18.out index de373df99003f9..26008cfa8ca2ef 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query18.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query18.out @@ -31,10 +31,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->c_current_addr_sk --------------------------------------PhysicalProject -----------------------------------------filter(c_birth_month IN (1, 10, 2, 4, 7, 8)) +----------------------------------------filter(customer.c_birth_month IN (1, 10, 2, 4, 7, 8)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(ca_state IN ('GA', 'IN', 'ME', 'NC', 'OK', 'WA', 'WY')) +----------------------------------------filter(customer_address.ca_state IN ('GA', 'IN', 'ME', 'NC', 'OK', 'WA', 'WY')) ------------------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalProject ----------------------------filter((date_dim.d_year = 1998)) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query19.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query19.out index 581fb8d1401f0c..0189846f162192 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query19.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 c_current_addr_sk->ca_address_sk ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query20.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query20.out index bdd3a0f81c2445..81d93c997643c1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query20.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query20.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2002-02-25') and (date_dim.d_date >= '2002-01-26')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Shoes', 'Women')) +----------------------------filter(item.i_category IN ('Books', 'Shoes', 'Women')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query21.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query21.out index ab3e69e8518c0e..5e04a2ee57a4c0 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query21.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query21.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) <= 1.5) and (if((inv_before > 0), (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) +--------filter(((cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)) <= 1.5) and (if((x.inv_before > 0), (cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query23.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query23.out index 5693c4b9fc7279..6cd435df3feb0f 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query23.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query23.out @@ -14,7 +14,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 ----------------------PhysicalProject -------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[item] @@ -27,7 +27,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------filter(( not ss_customer_sk IS NULL)) +--------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------PhysicalOlapScan[store_sales] ----------PhysicalProject ------------hashAgg[GLOBAL] @@ -40,10 +40,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ------------------------------PhysicalProject ---------------------------------filter(( not ss_customer_sk IS NULL)) +--------------------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ------------------------------PhysicalProject ---------------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query24.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query24.out index 9c2fb5d54823fb..ad35c370a7a522 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query24.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query24.out @@ -20,7 +20,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------filter((store.s_market_id = 8)) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (customer.c_birth_country = upper(customer_address.ca_country)))) ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer] ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query27.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query27.out index a78f43e684258a..329da903a9e14a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query27.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query27.out @@ -28,6 +28,6 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(s_state IN ('AL', 'LA', 'MI', 'MO', 'SC', 'TN')) +------------------------filter(store.s_state IN ('AL', 'LA', 'MI', 'MO', 'SC', 'TN')) --------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query29.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query29.out index 09ee0329c24b24..c06fbe9be99eee 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query29.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query29.out @@ -38,6 +38,6 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store] ------------------PhysicalProject ---------------------filter(d_year IN (1999, 2000, 2001)) +--------------------filter(d3.d_year IN (1999, 2000, 2001)) ----------------------PhysicalOlapScan[date_dim(d3)] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query30.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query30.out index b6282993b41245..b552ee11d7eed6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query30.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query30.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query31.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query31.out index ff37e1b29f4732..41f1563d16d090 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query31.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query31.out @@ -16,7 +16,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 --------------------PhysicalProject -----------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +----------------------filter((ss.d_year = 2000) and ss.d_qoy IN (1, 2, 3)) ------------------------PhysicalOlapScan[date_dim] ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 ----------------------PhysicalProject -------------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) +------------------------filter((ws.d_year = 2000) and ws.d_qoy IN (1, 2, 3)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] @@ -45,12 +45,12 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalQuickSort[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF10 ca_county->ca_county +--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((ws2.web_sales > 0.00), (cast(ws3.web_sales as DECIMALV3(38, 8)) / ws2.web_sales), NULL) > if((ss2.store_sales > 0.00), (cast(ss3.store_sales as DECIMALV3(38, 8)) / ss2.store_sales), NULL))) build RFs:RF10 ca_county->ca_county ----------------PhysicalProject ------------------filter((ws3.d_qoy = 3) and (ws3.d_year = 2000)) --------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF10 ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF9 ca_county->ca_county +------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((ws1.web_sales > 0.00), (cast(ws2.web_sales as DECIMALV3(38, 8)) / ws1.web_sales), NULL) > if((ss1.store_sales > 0.00), (cast(ss2.store_sales as DECIMALV3(38, 8)) / ss1.store_sales), NULL))) build RFs:RF9 ca_county->ca_county --------------------PhysicalProject ----------------------filter((ws2.d_qoy = 2) and (ws2.d_year = 2000)) ------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF9 diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query32.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query32.out index ef8807bfdbf3ef..e252239b8dfac2 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query32.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query32.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +--------------filter((cast(catalog_sales.cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) ----------------PhysicalWindow ------------------PhysicalQuickSort[LOCAL_SORT] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query34.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query34.out index dc2e92a063f466..22fb630fe08bf1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query34.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query34.out @@ -19,13 +19,13 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000)) +----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('0-500', '1001-5000')) +------------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('0-500', '1001-5000')) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject ---------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County')) +--------------------------filter(store.s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County')) ----------------------------PhysicalOlapScan[store] ------------PhysicalProject --------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query36.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query36.out index 75e000b14a2d46..8e58dba936856d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query36.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query36.out @@ -28,6 +28,6 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(s_state IN ('AL', 'GA', 'MI', 'MO', 'OH', 'SC', 'SD', 'TN')) +----------------------------------filter(store.s_state IN ('AL', 'GA', 'MI', 'MO', 'OH', 'SC', 'SD', 'TN')) ------------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query37.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query37.out index dec1c0455e5db0..92ac2f1d9e5d2b 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query37.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query37.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856)) +----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and item.i_manufact_id IN (1000, 707, 747, 856)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21')) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query39.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query39.out index 2221fd7647d2cc..3b5eded69f0405 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query39.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter(( not (mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) +------filter(( not (foo.mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) --------hashAgg[GLOBAL] ----------PhysicalProject ------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=() @@ -13,7 +13,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->inv_date_sk ----------------------PhysicalOlapScan[inventory] apply RFs: RF0 ----------------------PhysicalProject -------------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2)) +------------------------filter((date_dim.d_year = 1998) and inv.d_moy IN (1, 2)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query4.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query4.out index 088261b87a10f0..0311e1eae66716 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query4.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query4.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_w_firstyear.year_total > 0.000000), (cast(t_w_secyear.year_total as DECIMALV3(38, 16)) / t_w_firstyear.year_total), NULL))) build RFs:RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF18 customer_id->c_customer_id;RF19 customer_id->c_customer_id;RF20 customer_id->c_customer_id;RF21 customer_id->c_customer_id ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF15 customer_id->c_customer_id;RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.000000), (cast(t_s_secyear.year_total as DECIMALV3(38, 16)) / t_s_firstyear.year_total), NULL))) build RFs:RF15 customer_id->c_customer_id;RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF12 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query41.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query41.out index c7c720dd75259e..12dad98df5b0fc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query41.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query41.out @@ -18,6 +18,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'medium'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('dodger', 'indian', 'spring', 'tan'),i_units IN ('Bunch', 'Carton', 'Tsp', 'Unknown'),OR[AND[i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp')],AND[i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown')]]],AND[i_size IN ('N/A', 'large'),i_color IN ('blue', 'chartreuse', 'peru', 'saddle'),i_units IN ('Each', 'Gram', 'Oz', 'Pallet'),OR[AND[i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz')],AND[i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet')]]]]],AND[(item.i_category = 'Women'),i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('aquamarine', 'blanched', 'gainsboro', 'tomato'),i_units IN ('Case', 'Dozen', 'Ounce', 'Tbl'),OR[AND[i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce')],AND[i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl')]]],AND[i_size IN ('extra large', 'small'),i_color IN ('almond', 'chiffon', 'lime', 'violet'),i_units IN ('Box', 'Dram', 'Pound', 'Ton'),OR[AND[i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton')],AND[i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram')]]]]]] and i_category IN ('Men', 'Women')) +------------------------------filter(OR[AND[(item.i_category = 'Men'),item.i_size IN ('N/A', 'economy', 'large', 'medium'),OR[AND[item.i_size IN ('economy', 'medium'),item.i_color IN ('dodger', 'indian', 'spring', 'tan'),item.i_units IN ('Bunch', 'Carton', 'Tsp', 'Unknown'),OR[AND[item.i_color IN ('dodger', 'tan'),item.i_units IN ('Bunch', 'Tsp')],AND[item.i_color IN ('indian', 'spring'),item.i_units IN ('Carton', 'Unknown')]]],AND[item.i_size IN ('N/A', 'large'),item.i_color IN ('blue', 'chartreuse', 'peru', 'saddle'),item.i_units IN ('Each', 'Gram', 'Oz', 'Pallet'),OR[AND[item.i_color IN ('blue', 'chartreuse'),item.i_units IN ('Each', 'Oz')],AND[item.i_color IN ('peru', 'saddle'),item.i_units IN ('Gram', 'Pallet')]]]]],AND[(item.i_category = 'Women'),item.i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[item.i_size IN ('economy', 'medium'),item.i_color IN ('aquamarine', 'blanched', 'gainsboro', 'tomato'),item.i_units IN ('Case', 'Dozen', 'Ounce', 'Tbl'),OR[AND[item.i_color IN ('aquamarine', 'gainsboro'),item.i_units IN ('Dozen', 'Ounce')],AND[item.i_color IN ('blanched', 'tomato'),item.i_units IN ('Case', 'Tbl')]]],AND[item.i_size IN ('extra large', 'small'),item.i_color IN ('almond', 'chiffon', 'lime', 'violet'),item.i_units IN ('Box', 'Dram', 'Pound', 'Ton'),OR[AND[item.i_color IN ('chiffon', 'violet'),item.i_units IN ('Pound', 'Ton')],AND[item.i_color IN ('almond', 'lime'),item.i_units IN ('Box', 'Dram')]]]]]] and item.i_category IN ('Men', 'Women')) --------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query44.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query44.out index 1672d01807d616..7d04ce966c7215 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query44.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query44.out @@ -36,7 +36,7 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 146) and store_sales.ss_addr_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF0 item_sk->i_item_sk @@ -66,6 +66,6 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 146) and store_sales.ss_addr_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query45.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query45.out index 8331822cb0cf53..7c83726a2a3a02 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query45.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query46.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query46.out index cb36bb6f94c425..8f1499d1908845 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query46.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query46.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) +----------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() ----------------PhysicalProject @@ -21,13 +21,13 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001)) +--------------------------------------filter(date_dim.d_dow IN (0, 6) and date_dim.d_year IN (1999, 2000, 2001)) ----------------------------------------PhysicalOlapScan[date_dim] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)]) ------------------------------------PhysicalOlapScan[household_demographics] ----------------------------PhysicalProject -------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove')) +------------------------------filter(store.s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject --------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query47.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query47.out index daae3903e87fab..bf32fcef03197c 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query47.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query47.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002)) +----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[item] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1)) and (v1.s_company_name = v1_lag.s_company_name) and (v1.s_store_name = v1_lag.s_store_name)) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 s_store_name->s_store_name;RF6 s_company_name->s_company_name;RF7 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF7 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query48.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query48.out index d6afe543a26207..4fce44a48ec4e8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query48.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query48.out @@ -9,17 +9,17 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[customer_address.ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[customer_address.ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->ss_cdemo_sk ------------------------PhysicalProject --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'College', 'Primary') and customer_demographics.cd_marital_status IN ('D', 'U', 'W')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) +----------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) ------------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject ------------------filter((date_dim.d_year = 1999)) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query53.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query53.out index 191dabd72c00ba..788730b48385c9 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query53.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query53.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)) +----------------------------------filter(date_dim.d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query56.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query56.out index b30c33ad0af8f1..45b5e27a93a127 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query56.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -75,7 +75,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query57.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query57.out index 6d74bb9827a198..8ff0ef11d5c9c8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query57.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query57.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000)) +----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[item] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.cc_name = v1_lag.cc_name) and (v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1))) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 cc_name->cc_name;RF6 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query58.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query58.out index 77b393da701ebe..eed0db6f7ae3ad 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query58.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query58.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id +------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id --------------PhysicalProject ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] @@ -33,7 +33,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF13 --------------PhysicalProject -----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id +----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id ------------------PhysicalProject --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query6.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query6.out index 731a4957603462..29661273abd76d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query6.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query6.out @@ -36,7 +36,7 @@ PhysicalResultSink --------------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2002)) ----------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) +--------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item(i)] ----------------------------------hashAgg[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query63.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query63.out index 5ddb6ea404d4c0..e86115f4f5e312 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query63.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query63.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192)) +----------------------------------filter(date_dim.d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query64.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query64.out index 11925fe9433783..0803ffa5c76d67 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query64.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query64.out @@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd1.cd_marital_status = cd2.cd_marital_status))) ------------------------------------------------PhysicalProject --------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() ----------------------------------------------------PhysicalProject @@ -56,7 +56,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ------------------------------------------------------------------------------------------PhysicalProject --------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF23 ------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------filter(d_year IN (2001, 2002)) +--------------------------------------------------------------------------filter(d1.d_year IN (2001, 2002)) ----------------------------------------------------------------------------PhysicalOlapScan[date_dim(d1)] --------------------------------------------------------------------PhysicalProject ----------------------------------------------------------------------PhysicalOlapScan[store] @@ -85,7 +85,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------PhysicalProject ----------------------PhysicalOlapScan[income_band(ib2)] ----------------PhysicalProject -------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) +------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and item.i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) --------------------PhysicalOlapScan[item] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query65.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query65.out index 1b565462305cf5..8cd4cea253eb1b 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query65.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query65.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(sc.revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->ss_store_sk --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query66.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query66.out index 13067d5ffc3547..606b90c3c9da37 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query66.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query66.out @@ -24,7 +24,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 RF2 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 1998)) @@ -48,7 +48,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5 RF6 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 1998)) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query68.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query68.out index 274f7d2c342f23..20f41d9ef8ab17 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query68.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address(current_addr)] apply RFs: RF5 ----------------PhysicalProject @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill')) +--------------------------------------filter(store.s_city IN ('Five Points', 'Pleasant Hill')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)]) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query69.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query69.out index 75bad3fb2e4d8a..9a9681b6403d55 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query69.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query69.out @@ -42,6 +42,6 @@ PhysicalResultSink --------------------------------------filter((date_dim.d_moy <= 3) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2000)) ----------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------filter(ca_state IN ('MI', 'TX', 'VA')) +--------------------------------filter(ca.ca_state IN ('MI', 'TX', 'VA')) ----------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query71.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query71.out index 4017acc7076150..72302b069957ad 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query71.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query71.out @@ -31,6 +31,6 @@ PhysicalResultSink --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998)) ----------------------------PhysicalOlapScan[date_dim] --------------------PhysicalProject -----------------------filter(t_meal_time IN ('breakfast', 'dinner')) +----------------------filter(time_dim.t_meal_time IN ('breakfast', 'dinner')) ------------------------PhysicalOlapScan[time_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query72.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query72.out index 1c72c25511ec93..e253c5160d6ec8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query72.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query72.out @@ -21,7 +21,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() --------------------------------------PhysicalProject -----------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d_date, 5))) +----------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d1.d_date, 5))) ------------------------------------------PhysicalProject --------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_cdemo_sk = customer_demographics.cd_demo_sk)) otherCondition=() build RFs:RF2 cd_demo_sk->cs_bill_cdemo_sk ----------------------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query73.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query73.out index 606b8de0f35551..2a73e139896d26 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query73.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query73.out @@ -21,12 +21,12 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('501-1000', 'Unknown')) +------------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('501-1000', 'Unknown')) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject ---------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Fairfield County', 'Walker County')) +--------------------------filter(store.s_county IN ('Barrow County', 'Daviess County', 'Fairfield County', 'Walker County')) ----------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query74.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query74.out index 5fb0fa4978fb2f..8941615acb6995 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query74.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query74.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((year_total > 0.0), (year_total / year_total), NULL) > if((year_total > 0.0), (year_total / year_total), NULL))) build RFs:RF11 c_customer_sk->ws_bill_customer_sk +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((t_w_firstyear.year_total > 0.0), (t_w_secyear.year_total / t_w_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.0), (t_s_secyear.year_total / t_s_firstyear.year_total), NULL))) build RFs:RF11 c_customer_sk->ws_bill_customer_sk ------------PhysicalProject --------------hashAgg[GLOBAL] ----------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query75.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query75.out index af882fec270033..46d3eef5f87005 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query75.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query75.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Home')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (1998, 1999)) +------------------------filter(date_dim.d_year IN (1998, 1999)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF6 ss_ticket_number->sr_ticket_number;RF7 ss_item_sk->sr_item_sk @@ -37,7 +37,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Home')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (1998, 1999)) +------------------------filter(date_dim.d_year IN (1998, 1999)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF10 ws_order_number->wr_order_number;RF11 ws_item_sk->wr_item_sk @@ -53,14 +53,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Home')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (1998, 1999)) +------------------------filter(date_dim.d_year IN (1998, 1999)) --------------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id +------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(curr_yr.sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(prev_yr.sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id --------------filter((curr_yr.d_year = 1999)) ----------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 RF13 RF14 RF15 --------------filter((prev_yr.d_year = 1998)) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query76.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query76.out index b0aff5d8d1f41f..f9be09c1483c38 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query76.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query76.out @@ -18,7 +18,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF0 --------------------------PhysicalProject -----------------------------filter(ss_hdemo_sk IS NULL) +----------------------------filter(store_sales.ss_hdemo_sk IS NULL) ------------------------------PhysicalOlapScan[store_sales] --------------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------------PhysicalProject @@ -26,7 +26,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF1 --------------------------PhysicalProject -----------------------------filter(ws_bill_addr_sk IS NULL) +----------------------------filter(web_sales.ws_bill_addr_sk IS NULL) ------------------------------PhysicalOlapScan[web_sales] --------------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------------PhysicalProject @@ -34,6 +34,6 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF2 --------------------------PhysicalProject -----------------------------filter(cs_warehouse_sk IS NULL) +----------------------------filter(catalog_sales.cs_warehouse_sk IS NULL) ------------------------------PhysicalOlapScan[catalog_sales] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query78.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query78.out index 9d12b1032063f2..3ef3609f1d0a6e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query78.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query78.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)]) +----------filter(OR[(coalesce(ws.ws_qty, 0) > 0),(coalesce(cs.cs_qty, 0) > 0)]) ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=() --------------PhysicalProject ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=() diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query79.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query79.out index 651eaddb672e93..bdd55bb0c722e9 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query79.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query79.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dow = 1) and d_year IN (1998, 1999, 2000)) +----------------------------------filter((date_dim.d_dow = 1) and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)]) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query8.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query8.out index 7b9dd484cb7b16..3c5c184941a0f9 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query8.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query8.out @@ -40,6 +40,6 @@ PhysicalResultSink ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +----------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ------------------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query81.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query81.out index 90eeacd2464d9e..9de000cdbec343 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query81.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query81.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query82.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query82.out index e3952bb90b9647..73bde84ab6d53d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query82.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query82.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639)) +----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and item.i_manufact_id IN (138, 169, 339, 639)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09')) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query83.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query83.out index 2f2d328bd588db..a63553b6c6bed4 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query83.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query83.out @@ -28,7 +28,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF8 ------------------------------------------PhysicalProject ---------------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +--------------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ----------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[item] apply RFs: RF12 RF13 @@ -53,7 +53,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF4 ------------------------------------------PhysicalProject ---------------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +--------------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ----------------------------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashAgg[GLOBAL] @@ -76,6 +76,6 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +----------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ------------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query85.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query85.out index bda79431d3fbd4..567a1d4c2c34b6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query85.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query85.out @@ -15,15 +15,15 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF5 wr_returning_cdemo_sk->cd_demo_sk;RF6 cd_marital_status->cd_marital_status;RF7 cd_education_status->cd_education_status ----------------------------PhysicalProject -------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W')) +------------------------------filter(cd2.cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd2.cd_marital_status IN ('M', 'S', 'W')) --------------------------------PhysicalOlapScan[customer_demographics(cd2)] apply RFs: RF5 RF6 RF7 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->cd_demo_sk --------------------------------PhysicalProject -----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W')) +----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd1.cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd1.cd_marital_status IN ('M', 'S', 'W')) ------------------------------------PhysicalOlapScan[customer_demographics(cd1)] apply RFs: RF4 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->wr_refunded_addr_sk +----------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[customer_address.ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->wr_refunded_addr_sk ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->wr_item_sk;RF2 ws_order_number->wr_order_number ----------------------------------------PhysicalProject @@ -37,7 +37,7 @@ PhysicalResultSink ----------------------------------------------filter((date_dim.d_year = 2000)) ------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) +--------------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) ----------------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------PhysicalOlapScan[reason] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query88.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query88.out index d5f545332555d4..56dcd5c5c49d87 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query88.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query88.out @@ -23,7 +23,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -43,7 +43,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -63,7 +63,7 @@ PhysicalResultSink ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30)) ------------------------------------PhysicalOlapScan[time_dim] ----------------------------PhysicalProject -------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((store.s_store_name = 'ese')) @@ -83,7 +83,7 @@ PhysicalResultSink --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30)) ----------------------------------PhysicalOlapScan[time_dim] --------------------------PhysicalProject -----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ------------------------------PhysicalOlapScan[household_demographics] ----------------------PhysicalProject ------------------------filter((store.s_store_name = 'ese')) @@ -103,7 +103,7 @@ PhysicalResultSink ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30)) --------------------------------PhysicalOlapScan[time_dim] ------------------------PhysicalProject ---------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((store.s_store_name = 'ese')) @@ -123,7 +123,7 @@ PhysicalResultSink ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30)) ------------------------------PhysicalOlapScan[time_dim] ----------------------PhysicalProject -------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------PhysicalOlapScan[household_demographics] ------------------PhysicalProject --------------------filter((store.s_store_name = 'ese')) @@ -143,7 +143,7 @@ PhysicalResultSink --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30)) ----------------------------PhysicalOlapScan[time_dim] --------------------PhysicalProject -----------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +----------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ------------------------PhysicalOlapScan[household_demographics] ----------------PhysicalProject ------------------filter((store.s_store_name = 'ese')) @@ -163,7 +163,7 @@ PhysicalResultSink ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30)) --------------------------PhysicalOlapScan[time_dim] ------------------PhysicalProject ---------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------PhysicalOlapScan[household_demographics] --------------PhysicalProject ----------------filter((store.s_store_name = 'ese')) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query89.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query89.out index 45741d4ce87714..21d97fe65a667d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query89.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query89.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (avg_monthly_sales = 0.0000)) and ((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) +------------filter(( not (tmp1.avg_monthly_sales = 0.0000)) and ((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------hashAgg[GLOBAL] @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious')) +--------------------------------------filter(OR[AND[item.i_category IN ('Electronics', 'Jewelry', 'Shoes'),item.i_class IN ('athletic', 'portable', 'semi-precious')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'maternity', 'rock')]] and item.i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and item.i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 1999)) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query91.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query91.out index b16e802f2e7086..8b6607a2f654ec 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query91.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query91.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = customer.c_current_cdemo_sk)) otherCondition=() build RFs:RF2 c_current_cdemo_sk->cd_demo_sk --------------------------------PhysicalProject -----------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +----------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF2 --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((household_demographics.hd_demo_sk = customer.c_current_hdemo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->c_current_hdemo_sk @@ -31,7 +31,7 @@ PhysicalResultSink ------------------------------------------filter((customer_address.ca_gmt_offset = -6.00)) --------------------------------------------PhysicalOlapScan[customer_address] ------------------------------------PhysicalProject ---------------------------------------filter((hd_buy_potential like '1001-5000%')) +--------------------------------------filter((household_demographics.hd_buy_potential like '1001-5000%')) ----------------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query92.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query92.out index b02739c84db1ba..b69e7b837d186d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query92.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query92.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +------------filter((cast(web_sales.ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query94.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query94.out index c5657bbc154bae..6be107b8333fc5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query94.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query94.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF3 ws_order_number->ws_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF3 ws_order_number->ws_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF3 --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query95.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query95.out index 304cca006a4d74..67d5aa669c7269 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query95.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF8 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query98.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query98.out index bc1091c292f35c..e334b81fa55857 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query98.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query98.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2002-06-19') and (date_dim.d_date >= '2002-05-20')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Music', 'Shoes', 'Sports')) +----------------------------filter(item.i_category IN ('Music', 'Shoes', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query1.out b/regression-test/data/shape_check/tpcds_sf100/shape/query1.out index 7235a18bb4c329..cbe360ef7fbbba 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query1.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query1.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalProject ----------------PhysicalOlapScan[customer] apply RFs: RF4 --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->ctr_store_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF2 diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query10.out b/regression-test/data/shape_check/tpcds_sf100/shape/query10.out index d9535f09438e5e..556775c934047f 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query10.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query10.out @@ -42,6 +42,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County')) +------------------------------------filter(ca.ca_county IN ('Cochran County', 'Kandiyohi County', 'Marquette County', 'Storey County', 'Warren County')) --------------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query11.out b/regression-test/data/shape_check/tpcds_sf100/shape/query11.out index 42741b2cefc301..802b6eeedff2fb 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query11.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query11.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000) > if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000))) build RFs:RF11 c_customer_sk->ws_bill_customer_sk +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(38, 8)) / t_w_firstyear.year_total), 0.000000) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(38, 8)) / t_s_firstyear.year_total), 0.000000))) build RFs:RF11 c_customer_sk->ws_bill_customer_sk ------------PhysicalProject --------------hashAgg[GLOBAL] ----------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query12.out b/regression-test/data/shape_check/tpcds_sf100/shape/query12.out index c7d13441a9337c..12af901e79fdbd 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query12.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query12.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '1998-05-06') and (date_dim.d_date >= '1998-04-06')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Men', 'Sports')) +----------------------------filter(item.i_category IN ('Books', 'Men', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query13.out b/regression-test/data/shape_check/tpcds_sf100/shape/query13.out index 0b92618522bdf0..dee3f67224b2b6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query13.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query13.out @@ -9,25 +9,25 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->cd_demo_sk ----------------PhysicalProject -------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')]] and cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and cd_marital_status IN ('D', 'M', 'S')) +------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '4 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Unknown')]] and customer_demographics.cd_education_status IN ('4 yr Degree', 'College', 'Unknown') and customer_demographics.cd_marital_status IN ('D', 'M', 'S')) --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->ss_hdemo_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ss_sold_date_sk ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->ss_addr_sk +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('KS', 'MI', 'SD'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('CO', 'MO', 'ND'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('NH', 'OH', 'TX'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->ss_addr_sk ----------------------------PhysicalProject ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4 ----------------------------PhysicalProject -------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'KS', 'MI', 'MO', 'ND', 'NH', 'OH', 'SD', 'TX')) +------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('CO', 'KS', 'MI', 'MO', 'ND', 'NH', 'OH', 'SD', 'TX')) --------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------filter((date_dim.d_year = 2001)) ----------------------------PhysicalOlapScan[date_dim] --------------------PhysicalProject -----------------------filter(hd_dep_count IN (1, 3)) +----------------------filter(household_demographics.hd_dep_count IN (1, 3)) ------------------------PhysicalOlapScan[household_demographics] ------------PhysicalProject --------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query15.out b/regression-test/data/shape_check/tpcds_sf100/shape/query15.out index 5955d4e3f51453..b0db4e92bb2bc0 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query15.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query15.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),customer_address.ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_sold_date_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query16.out b/regression-test/data/shape_check/tpcds_sf100/shape/query16.out index 3b148bf6fc8b24..1c4377348abffb 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query16.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query16.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF3 cs_order_number->cs_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF3 cs_order_number->cs_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF3 --------------------PhysicalProject @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------------filter((date_dim.d_date <= '2002-05-31') and (date_dim.d_date >= '2002-04-01')) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject ---------------------------filter(cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County')) +--------------------------filter(call_center.cc_county IN ('Barrow County', 'Daviess County', 'Luce County', 'Richland County', 'Ziebach County')) ----------------------------PhysicalOlapScan[call_center] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query17.out b/regression-test/data/shape_check/tpcds_sf100/shape/query17.out index a0034e294d3d41..5f897b6e19a6e4 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query17.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF8 RF9 RF10 ------------------------PhysicalProject ---------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------filter(d3.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------PhysicalOlapScan[date_dim(d3)] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF7 s_store_sk->ss_store_sk @@ -35,7 +35,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0 RF6 ------------------------------------PhysicalProject ---------------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------------------filter(d2.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------------------PhysicalOlapScan[date_dim(d2)] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query18.out b/regression-test/data/shape_check/tpcds_sf100/shape/query18.out index de373df99003f9..26008cfa8ca2ef 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query18.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query18.out @@ -31,10 +31,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->c_current_addr_sk --------------------------------------PhysicalProject -----------------------------------------filter(c_birth_month IN (1, 10, 2, 4, 7, 8)) +----------------------------------------filter(customer.c_birth_month IN (1, 10, 2, 4, 7, 8)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(ca_state IN ('GA', 'IN', 'ME', 'NC', 'OK', 'WA', 'WY')) +----------------------------------------filter(customer_address.ca_state IN ('GA', 'IN', 'ME', 'NC', 'OK', 'WA', 'WY')) ------------------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalProject ----------------------------filter((date_dim.d_year = 1998)) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query19.out b/regression-test/data/shape_check/tpcds_sf100/shape/query19.out index f5f4ec43f09647..b5a9d58710d8d8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query19.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 c_current_addr_sk->ca_address_sk ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query20.out b/regression-test/data/shape_check/tpcds_sf100/shape/query20.out index bdd3a0f81c2445..81d93c997643c1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query20.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query20.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2002-02-25') and (date_dim.d_date >= '2002-01-26')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Shoes', 'Women')) +----------------------------filter(item.i_category IN ('Books', 'Shoes', 'Women')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query21.out b/regression-test/data/shape_check/tpcds_sf100/shape/query21.out index fa3be9c477386d..bb5a8d7516d504 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query21.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query21.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) <= 1.5) and (if((inv_before > 0), (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) +--------filter(((cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)) <= 1.5) and (if((x.inv_before > 0), (cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query23.out b/regression-test/data/shape_check/tpcds_sf100/shape/query23.out index 67ee2f066b0760..b6c70b2729f27a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query23.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query23.out @@ -14,7 +14,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------PhysicalProject -------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[item] @@ -27,7 +27,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------filter(( not ss_customer_sk IS NULL)) +--------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------PhysicalOlapScan[store_sales] ----------PhysicalProject ------------hashAgg[GLOBAL] @@ -40,10 +40,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ------------------------------PhysicalProject ---------------------------------filter(( not ss_customer_sk IS NULL)) +--------------------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ------------------------------PhysicalProject ---------------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query24.out b/regression-test/data/shape_check/tpcds_sf100/shape/query24.out index 124d59fe06da11..dd9b63b7c62d32 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query24.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query24.out @@ -20,7 +20,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------filter((store.s_market_id = 8)) --------------------------------PhysicalOlapScan[store] apply RFs: RF2 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 ca_address_sk->c_current_addr_sk +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (customer.c_birth_country = upper(customer_address.ca_country)))) build RFs:RF0 ca_address_sk->c_current_addr_sk ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query27.out b/regression-test/data/shape_check/tpcds_sf100/shape/query27.out index 939e6ac6c2cfef..6f7e936f481d97 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query27.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query27.out @@ -28,6 +28,6 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(s_state IN ('AL', 'LA', 'MI', 'MO', 'SC', 'TN')) +------------------------filter(store.s_state IN ('AL', 'LA', 'MI', 'MO', 'SC', 'TN')) --------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query29.out b/regression-test/data/shape_check/tpcds_sf100/shape/query29.out index 95641d33c312de..25ded95ade2973 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query29.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query29.out @@ -38,6 +38,6 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store] ------------------PhysicalProject ---------------------filter(d_year IN (1999, 2000, 2001)) +--------------------filter(d3.d_year IN (1999, 2000, 2001)) ----------------------PhysicalOlapScan[date_dim(d3)] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query30.out b/regression-test/data/shape_check/tpcds_sf100/shape/query30.out index 360e33be49255a..fe3fb69cd26d55 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query30.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query30.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query31.out b/regression-test/data/shape_check/tpcds_sf100/shape/query31.out index 41365ae89c3bed..d7ac2568703414 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query31.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query31.out @@ -16,7 +16,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 --------------------PhysicalProject -----------------------filter((ss.d_year = 2000) and d_qoy IN (1, 2, 3)) +----------------------filter((ss.d_year = 2000) and ss.d_qoy IN (1, 2, 3)) ------------------------PhysicalOlapScan[date_dim] ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 ----------------------PhysicalProject -------------------------filter((ws.d_year = 2000) and d_qoy IN (1, 2, 3)) +------------------------filter((ws.d_year = 2000) and ws.d_qoy IN (1, 2, 3)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] @@ -45,12 +45,12 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalQuickSort[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF10 ca_county->ca_county +--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((ws2.web_sales > 0.00), (cast(ws3.web_sales as DECIMALV3(38, 8)) / ws2.web_sales), NULL) > if((ss2.store_sales > 0.00), (cast(ss3.store_sales as DECIMALV3(38, 8)) / ss2.store_sales), NULL))) build RFs:RF10 ca_county->ca_county ----------------PhysicalProject ------------------filter((ws3.d_qoy = 3) and (ws3.d_year = 2000)) --------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF10 ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF9 ca_county->ca_county +------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((ws1.web_sales > 0.00), (cast(ws2.web_sales as DECIMALV3(38, 8)) / ws1.web_sales), NULL) > if((ss1.store_sales > 0.00), (cast(ss2.store_sales as DECIMALV3(38, 8)) / ss1.store_sales), NULL))) build RFs:RF9 ca_county->ca_county --------------------PhysicalProject ----------------------filter((ws2.d_qoy = 2) and (ws2.d_year = 2000)) ------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF9 diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query32.out b/regression-test/data/shape_check/tpcds_sf100/shape/query32.out index ef8807bfdbf3ef..e252239b8dfac2 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query32.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query32.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +--------------filter((cast(catalog_sales.cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) ----------------PhysicalWindow ------------------PhysicalQuickSort[LOCAL_SORT] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query34.out b/regression-test/data/shape_check/tpcds_sf100/shape/query34.out index e9e3a75e403a59..bd1d90fcf21681 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query34.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query34.out @@ -19,13 +19,13 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (1998, 1999, 2000)) +----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('0-500', '1001-5000')) +------------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('0-500', '1001-5000')) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject ---------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County')) +--------------------------filter(store.s_county IN ('Barrow County', 'Daviess County', 'Franklin Parish', 'Luce County', 'Richland County', 'Walker County', 'Williamson County', 'Ziebach County')) ----------------------------PhysicalOlapScan[store] ------------PhysicalProject --------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query36.out b/regression-test/data/shape_check/tpcds_sf100/shape/query36.out index 8dbaa2718a447a..e447bb9cfc1438 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query36.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query36.out @@ -28,6 +28,6 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(s_state IN ('AL', 'GA', 'MI', 'MO', 'OH', 'SC', 'SD', 'TN')) +----------------------------------filter(store.s_state IN ('AL', 'GA', 'MI', 'MO', 'OH', 'SC', 'SD', 'TN')) ------------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query37.out b/regression-test/data/shape_check/tpcds_sf100/shape/query37.out index dec1c0455e5db0..92ac2f1d9e5d2b 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query37.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query37.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856)) +----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and item.i_manufact_id IN (1000, 707, 747, 856)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21')) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query39.out b/regression-test/data/shape_check/tpcds_sf100/shape/query39.out index 278a78844da920..0260d6a03fdcda 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query39.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter(( not (mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) +------filter(( not (foo.mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) --------hashAgg[GLOBAL] ----------PhysicalProject ------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=() build RFs:RF2 w_warehouse_sk->inv_warehouse_sk @@ -13,7 +13,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->inv_date_sk ----------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 RF2 ----------------------PhysicalProject -------------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2)) +------------------------filter((date_dim.d_year = 1998) and inv.d_moy IN (1, 2)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query4.out b/regression-test/data/shape_check/tpcds_sf100/shape/query4.out index 479aef2c34a707..41ffa4481da213 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query4.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query4.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_w_firstyear.year_total > 0.000000), (cast(t_w_secyear.year_total as DECIMALV3(38, 16)) / t_w_firstyear.year_total), NULL))) build RFs:RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF18 customer_id->c_customer_id;RF19 customer_id->c_customer_id;RF20 customer_id->c_customer_id;RF21 customer_id->c_customer_id ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF15 customer_id->c_customer_id;RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.000000), (cast(t_s_secyear.year_total as DECIMALV3(38, 16)) / t_s_firstyear.year_total), NULL))) build RFs:RF15 customer_id->c_customer_id;RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF12 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query41.out b/regression-test/data/shape_check/tpcds_sf100/shape/query41.out index c7c720dd75259e..12dad98df5b0fc 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query41.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query41.out @@ -18,6 +18,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'medium'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('dodger', 'indian', 'spring', 'tan'),i_units IN ('Bunch', 'Carton', 'Tsp', 'Unknown'),OR[AND[i_color IN ('dodger', 'tan'),i_units IN ('Bunch', 'Tsp')],AND[i_color IN ('indian', 'spring'),i_units IN ('Carton', 'Unknown')]]],AND[i_size IN ('N/A', 'large'),i_color IN ('blue', 'chartreuse', 'peru', 'saddle'),i_units IN ('Each', 'Gram', 'Oz', 'Pallet'),OR[AND[i_color IN ('blue', 'chartreuse'),i_units IN ('Each', 'Oz')],AND[i_color IN ('peru', 'saddle'),i_units IN ('Gram', 'Pallet')]]]]],AND[(item.i_category = 'Women'),i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[i_size IN ('economy', 'medium'),i_color IN ('aquamarine', 'blanched', 'gainsboro', 'tomato'),i_units IN ('Case', 'Dozen', 'Ounce', 'Tbl'),OR[AND[i_color IN ('aquamarine', 'gainsboro'),i_units IN ('Dozen', 'Ounce')],AND[i_color IN ('blanched', 'tomato'),i_units IN ('Case', 'Tbl')]]],AND[i_size IN ('extra large', 'small'),i_color IN ('almond', 'chiffon', 'lime', 'violet'),i_units IN ('Box', 'Dram', 'Pound', 'Ton'),OR[AND[i_color IN ('chiffon', 'violet'),i_units IN ('Pound', 'Ton')],AND[i_color IN ('almond', 'lime'),i_units IN ('Box', 'Dram')]]]]]] and i_category IN ('Men', 'Women')) +------------------------------filter(OR[AND[(item.i_category = 'Men'),item.i_size IN ('N/A', 'economy', 'large', 'medium'),OR[AND[item.i_size IN ('economy', 'medium'),item.i_color IN ('dodger', 'indian', 'spring', 'tan'),item.i_units IN ('Bunch', 'Carton', 'Tsp', 'Unknown'),OR[AND[item.i_color IN ('dodger', 'tan'),item.i_units IN ('Bunch', 'Tsp')],AND[item.i_color IN ('indian', 'spring'),item.i_units IN ('Carton', 'Unknown')]]],AND[item.i_size IN ('N/A', 'large'),item.i_color IN ('blue', 'chartreuse', 'peru', 'saddle'),item.i_units IN ('Each', 'Gram', 'Oz', 'Pallet'),OR[AND[item.i_color IN ('blue', 'chartreuse'),item.i_units IN ('Each', 'Oz')],AND[item.i_color IN ('peru', 'saddle'),item.i_units IN ('Gram', 'Pallet')]]]]],AND[(item.i_category = 'Women'),item.i_size IN ('economy', 'extra large', 'medium', 'small'),OR[AND[item.i_size IN ('economy', 'medium'),item.i_color IN ('aquamarine', 'blanched', 'gainsboro', 'tomato'),item.i_units IN ('Case', 'Dozen', 'Ounce', 'Tbl'),OR[AND[item.i_color IN ('aquamarine', 'gainsboro'),item.i_units IN ('Dozen', 'Ounce')],AND[item.i_color IN ('blanched', 'tomato'),item.i_units IN ('Case', 'Tbl')]]],AND[item.i_size IN ('extra large', 'small'),item.i_color IN ('almond', 'chiffon', 'lime', 'violet'),item.i_units IN ('Box', 'Dram', 'Pound', 'Ton'),OR[AND[item.i_color IN ('chiffon', 'violet'),item.i_units IN ('Pound', 'Ton')],AND[item.i_color IN ('almond', 'lime'),item.i_units IN ('Box', 'Dram')]]]]]] and item.i_category IN ('Men', 'Women')) --------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query44.out b/regression-test/data/shape_check/tpcds_sf100/shape/query44.out index 1672d01807d616..7d04ce966c7215 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query44.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query44.out @@ -36,7 +36,7 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 146) and store_sales.ss_addr_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF0 item_sk->i_item_sk @@ -66,6 +66,6 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 146) and store_sales.ss_addr_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query45.out b/regression-test/data/shape_check/tpcds_sf100/shape/query45.out index e273fad514ec36..26f74bf0cfa2d0 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query45.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->ws_bill_customer_sk @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query46.out b/regression-test/data/shape_check/tpcds_sf100/shape/query46.out index 9ac9c826d05dfc..2345d72dc3a565 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query46.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query46.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk +----------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->ss_customer_sk ----------------PhysicalProject @@ -21,13 +21,13 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 RF4 ------------------------------------PhysicalProject ---------------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001)) +--------------------------------------filter(date_dim.d_dow IN (0, 6) and date_dim.d_year IN (1999, 2000, 2001)) ----------------------------------------PhysicalOlapScan[date_dim] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 6),(household_demographics.hd_vehicle_count = 0)]) ------------------------------------PhysicalOlapScan[household_demographics] ----------------------------PhysicalProject -------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove')) +------------------------------filter(store.s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject --------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query47.out b/regression-test/data/shape_check/tpcds_sf100/shape/query47.out index 4836265d202c95..4f35cda8f278d6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query47.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query47.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002)) +----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[item] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1)) and (v1.s_company_name = v1_lag.s_company_name) and (v1.s_store_name = v1_lag.s_store_name)) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 s_store_name->s_store_name;RF6 s_company_name->s_company_name;RF7 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF7 RF9 RF11 RF13 RF15 RF17 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8 RF10 RF12 RF14 RF16 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query48.out b/regression-test/data/shape_check/tpcds_sf100/shape/query48.out index bd9309d833731c..ff8b2fe617b270 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query48.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query48.out @@ -9,17 +9,17 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IA', 'MD', 'MN'),(store_sales.ss_net_profit <= 2000.00)],AND[customer_address.ca_state IN ('IL', 'TX', 'VA'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[customer_address.ca_state IN ('IN', 'MI', 'WI'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->ss_cdemo_sk ------------------------PhysicalProject --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'U', 'W')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = '2 yr Degree')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'College', 'Primary') and customer_demographics.cd_marital_status IN ('D', 'U', 'W')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) +----------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) ------------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject ------------------filter((date_dim.d_year = 1999)) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query53.out b/regression-test/data/shape_check/tpcds_sf100/shape/query53.out index 1760d5b36125a7..e8a45a138a538d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query53.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query53.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)) +----------------------------------filter(date_dim.d_month_seq IN (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query56.out b/regression-test/data/shape_check/tpcds_sf100/shape/query56.out index b30c33ad0af8f1..45b5e27a93a127 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query56.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -75,7 +75,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('cyan', 'green', 'powder')) +------------------------------------filter(item.i_color IN ('cyan', 'green', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query57.out b/regression-test/data/shape_check/tpcds_sf100/shape/query57.out index 3ce6f46e0057f5..23ab041feb5dac 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query57.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query57.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000)) +----------------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[item] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.cc_name = v1_lag.cc_name) and (v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1))) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 cc_name->cc_name;RF6 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF8 RF10 RF12 RF14 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7 RF9 RF11 RF13 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query58.out b/regression-test/data/shape_check/tpcds_sf100/shape/query58.out index 8cbab3a197cab8..512f1cfd353c43 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query58.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query58.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id +------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id --------------PhysicalProject ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] @@ -33,7 +33,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF13 --------------PhysicalProject -----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id +----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id ------------------PhysicalProject --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query6.out b/regression-test/data/shape_check/tpcds_sf100/shape/query6.out index bab8096426d236..118ac1a7acc393 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query6.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query6.out @@ -36,7 +36,7 @@ PhysicalResultSink --------------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2002)) ----------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category +--------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item(i)] apply RFs: RF0 ----------------------------------hashAgg[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query63.out b/regression-test/data/shape_check/tpcds_sf100/shape/query63.out index 2d0eb1e9ffce4e..177209b2335f41 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query63.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query63.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192)) +----------------------------------filter(date_dim.d_month_seq IN (1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query64.out b/regression-test/data/shape_check/tpcds_sf100/shape/query64.out index 2d2a21e19171df..9e6927400f8dc1 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query64.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query64.out @@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF13 p_promo_sk->ss_promo_sk --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF12 cd_demo_sk->c_current_cdemo_sk +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd1.cd_marital_status = cd2.cd_marital_status))) build RFs:RF12 cd_demo_sk->c_current_cdemo_sk ------------------------------------------------PhysicalProject --------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() build RFs:RF11 cd_demo_sk->ss_cdemo_sk ----------------------------------------------------PhysicalProject @@ -56,7 +56,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ------------------------------------------------------------------------------------------PhysicalProject --------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF23 ------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------filter(d_year IN (2001, 2002)) +--------------------------------------------------------------------------filter(d1.d_year IN (2001, 2002)) ----------------------------------------------------------------------------PhysicalOlapScan[date_dim(d1)] --------------------------------------------------------------------PhysicalProject ----------------------------------------------------------------------PhysicalOlapScan[store] @@ -85,7 +85,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------PhysicalProject ----------------------PhysicalOlapScan[income_band(ib2)] ----------------PhysicalProject -------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) +------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and item.i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) --------------------PhysicalOlapScan[item] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query65.out b/regression-test/data/shape_check/tpcds_sf100/shape/query65.out index d52be20fbe52e0..8a0c0e54f87ca0 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query65.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query65.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(sc.revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->ss_store_sk --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query66.out b/regression-test/data/shape_check/tpcds_sf100/shape/query66.out index 79fbabca9a5cbd..0cb1052d38d29a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query66.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query66.out @@ -24,7 +24,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 1998)) @@ -48,7 +48,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5 RF6 RF7 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('GREAT EASTERN', 'LATVIAN')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 1998)) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query68.out b/regression-test/data/shape_check/tpcds_sf100/shape/query68.out index 274f7d2c342f23..20f41d9ef8ab17 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query68.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address(current_addr)] apply RFs: RF5 ----------------PhysicalProject @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill')) +--------------------------------------filter(store.s_city IN ('Five Points', 'Pleasant Hill')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = -1)]) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query69.out b/regression-test/data/shape_check/tpcds_sf100/shape/query69.out index 75bad3fb2e4d8a..9a9681b6403d55 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query69.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query69.out @@ -42,6 +42,6 @@ PhysicalResultSink --------------------------------------filter((date_dim.d_moy <= 3) and (date_dim.d_moy >= 1) and (date_dim.d_year = 2000)) ----------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------filter(ca_state IN ('MI', 'TX', 'VA')) +--------------------------------filter(ca.ca_state IN ('MI', 'TX', 'VA')) ----------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query71.out b/regression-test/data/shape_check/tpcds_sf100/shape/query71.out index 4017acc7076150..72302b069957ad 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query71.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query71.out @@ -31,6 +31,6 @@ PhysicalResultSink --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 1998)) ----------------------------PhysicalOlapScan[date_dim] --------------------PhysicalProject -----------------------filter(t_meal_time IN ('breakfast', 'dinner')) +----------------------filter(time_dim.t_meal_time IN ('breakfast', 'dinner')) ------------------------PhysicalOlapScan[time_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query72.out b/regression-test/data/shape_check/tpcds_sf100/shape/query72.out index 22742b1f2dc2c7..d9eb59818eb8e7 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query72.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query72.out @@ -21,7 +21,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->cs_item_sk --------------------------------------PhysicalProject -----------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d_date, 5))) build RFs:RF3 d_date_sk->cs_ship_date_sk +----------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d1.d_date, 5))) build RFs:RF3 d_date_sk->cs_ship_date_sk ------------------------------------------PhysicalProject --------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_cdemo_sk = customer_demographics.cd_demo_sk)) otherCondition=() build RFs:RF2 cd_demo_sk->cs_bill_cdemo_sk ----------------------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query73.out b/regression-test/data/shape_check/tpcds_sf100/shape/query73.out index 606b8de0f35551..2a73e139896d26 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query73.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query73.out @@ -21,12 +21,12 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('501-1000', 'Unknown')) +------------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('501-1000', 'Unknown')) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject ---------------------------filter(s_county IN ('Barrow County', 'Daviess County', 'Fairfield County', 'Walker County')) +--------------------------filter(store.s_county IN ('Barrow County', 'Daviess County', 'Fairfield County', 'Walker County')) ----------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query74.out b/regression-test/data/shape_check/tpcds_sf100/shape/query74.out index 5fb0fa4978fb2f..8941615acb6995 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query74.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query74.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((year_total > 0.0), (year_total / year_total), NULL) > if((year_total > 0.0), (year_total / year_total), NULL))) build RFs:RF11 c_customer_sk->ws_bill_customer_sk +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((t_w_firstyear.year_total > 0.0), (t_w_secyear.year_total / t_w_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.0), (t_s_secyear.year_total / t_s_firstyear.year_total), NULL))) build RFs:RF11 c_customer_sk->ws_bill_customer_sk ------------PhysicalProject --------------hashAgg[GLOBAL] ----------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query75.out b/regression-test/data/shape_check/tpcds_sf100/shape/query75.out index af882fec270033..46d3eef5f87005 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query75.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query75.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Home')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (1998, 1999)) +------------------------filter(date_dim.d_year IN (1998, 1999)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF6 ss_ticket_number->sr_ticket_number;RF7 ss_item_sk->sr_item_sk @@ -37,7 +37,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Home')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (1998, 1999)) +------------------------filter(date_dim.d_year IN (1998, 1999)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF10 ws_order_number->wr_order_number;RF11 ws_item_sk->wr_item_sk @@ -53,14 +53,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Home')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (1998, 1999)) +------------------------filter(date_dim.d_year IN (1998, 1999)) --------------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id +------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(curr_yr.sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(prev_yr.sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id --------------filter((curr_yr.d_year = 1999)) ----------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 RF13 RF14 RF15 --------------filter((prev_yr.d_year = 1998)) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query76.out b/regression-test/data/shape_check/tpcds_sf100/shape/query76.out index b0aff5d8d1f41f..f9be09c1483c38 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query76.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query76.out @@ -18,7 +18,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF0 --------------------------PhysicalProject -----------------------------filter(ss_hdemo_sk IS NULL) +----------------------------filter(store_sales.ss_hdemo_sk IS NULL) ------------------------------PhysicalOlapScan[store_sales] --------------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------------PhysicalProject @@ -26,7 +26,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF1 --------------------------PhysicalProject -----------------------------filter(ws_bill_addr_sk IS NULL) +----------------------------filter(web_sales.ws_bill_addr_sk IS NULL) ------------------------------PhysicalOlapScan[web_sales] --------------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------------PhysicalProject @@ -34,6 +34,6 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF2 --------------------------PhysicalProject -----------------------------filter(cs_warehouse_sk IS NULL) +----------------------------filter(catalog_sales.cs_warehouse_sk IS NULL) ------------------------------PhysicalOlapScan[catalog_sales] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query78.out b/regression-test/data/shape_check/tpcds_sf100/shape/query78.out index 9d12b1032063f2..3ef3609f1d0a6e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query78.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query78.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)]) +----------filter(OR[(coalesce(ws.ws_qty, 0) > 0),(coalesce(cs.cs_qty, 0) > 0)]) ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=() --------------PhysicalProject ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=() diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query79.out b/regression-test/data/shape_check/tpcds_sf100/shape/query79.out index 77381b2413e82c..ea7c27976d615f 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query79.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query79.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dow = 1) and d_year IN (1998, 1999, 2000)) +----------------------------------filter((date_dim.d_dow = 1) and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 5),(household_demographics.hd_vehicle_count > 4)]) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query8.out b/regression-test/data/shape_check/tpcds_sf100/shape/query8.out index e6b75f941a6c06..1e188ec784a6b4 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query8.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query8.out @@ -40,6 +40,6 @@ PhysicalResultSink ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +----------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ------------------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query81.out b/regression-test/data/shape_check/tpcds_sf100/shape/query81.out index a50e9c7e061b2a..e6a6a5f56364f6 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query81.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query81.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query82.out b/regression-test/data/shape_check/tpcds_sf100/shape/query82.out index e3952bb90b9647..73bde84ab6d53d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query82.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query82.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639)) +----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and item.i_manufact_id IN (138, 169, 339, 639)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09')) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query83.out b/regression-test/data/shape_check/tpcds_sf100/shape/query83.out index 2f2d328bd588db..a63553b6c6bed4 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query83.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query83.out @@ -28,7 +28,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF8 ------------------------------------------PhysicalProject ---------------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +--------------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ----------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[item] apply RFs: RF12 RF13 @@ -53,7 +53,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF4 ------------------------------------------PhysicalProject ---------------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +--------------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ----------------------------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashAgg[GLOBAL] @@ -76,6 +76,6 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) +----------------------------------------filter(date_dim.d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) ------------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query85.out b/regression-test/data/shape_check/tpcds_sf100/shape/query85.out index bb1519437b0c95..1ef911a0344165 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query85.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query85.out @@ -15,15 +15,15 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF5 wr_returning_cdemo_sk->cd_demo_sk;RF6 cd_marital_status->cd_marital_status;RF7 cd_education_status->cd_education_status ----------------------------PhysicalProject -------------------------------filter(cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W')) +------------------------------filter(cd2.cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd2.cd_marital_status IN ('M', 'S', 'W')) --------------------------------PhysicalOlapScan[customer_demographics(cd2)] apply RFs: RF5 RF6 RF7 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF4 wr_refunded_cdemo_sk->cd_demo_sk --------------------------------PhysicalProject -----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('M', 'S', 'W')) +----------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'Secondary')],AND[(cd1.cd_marital_status = 'W'),(cd1.cd_education_status = 'Advanced Degree')]] and cd1.cd_education_status IN ('4 yr Degree', 'Advanced Degree', 'Secondary') and cd1.cd_marital_status IN ('M', 'S', 'W')) ------------------------------------PhysicalOlapScan[customer_demographics(cd1)] apply RFs: RF4 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->wr_refunded_addr_sk +----------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('DE', 'FL', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[customer_address.ca_state IN ('ID', 'IN', 'ND'),(web_sales.ws_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IL', 'MT', 'OH'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->wr_refunded_addr_sk ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->wr_item_sk;RF2 ws_order_number->wr_order_number ----------------------------------------PhysicalProject @@ -37,7 +37,7 @@ PhysicalResultSink ----------------------------------------------filter((date_dim.d_year = 2000)) ------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) +--------------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) ----------------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------PhysicalOlapScan[reason] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query88.out b/regression-test/data/shape_check/tpcds_sf100/shape/query88.out index d5f545332555d4..56dcd5c5c49d87 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query88.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query88.out @@ -23,7 +23,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -43,7 +43,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -63,7 +63,7 @@ PhysicalResultSink ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30)) ------------------------------------PhysicalOlapScan[time_dim] ----------------------------PhysicalProject -------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((store.s_store_name = 'ese')) @@ -83,7 +83,7 @@ PhysicalResultSink --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30)) ----------------------------------PhysicalOlapScan[time_dim] --------------------------PhysicalProject -----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ------------------------------PhysicalOlapScan[household_demographics] ----------------------PhysicalProject ------------------------filter((store.s_store_name = 'ese')) @@ -103,7 +103,7 @@ PhysicalResultSink ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30)) --------------------------------PhysicalOlapScan[time_dim] ------------------------PhysicalProject ---------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((store.s_store_name = 'ese')) @@ -123,7 +123,7 @@ PhysicalResultSink ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30)) ------------------------------PhysicalOlapScan[time_dim] ----------------------PhysicalProject -------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) --------------------------PhysicalOlapScan[household_demographics] ------------------PhysicalProject --------------------filter((store.s_store_name = 'ese')) @@ -143,7 +143,7 @@ PhysicalResultSink --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30)) ----------------------------PhysicalOlapScan[time_dim] --------------------PhysicalProject -----------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +----------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ------------------------PhysicalOlapScan[household_demographics] ----------------PhysicalProject ------------------filter((store.s_store_name = 'ese')) @@ -163,7 +163,7 @@ PhysicalResultSink ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30)) --------------------------PhysicalOlapScan[time_dim] ------------------PhysicalProject ---------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (-1, 3, 4)) +--------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (-1, 3, 4)) ----------------------PhysicalOlapScan[household_demographics] --------------PhysicalProject ----------------filter((store.s_store_name = 'ese')) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query89.out b/regression-test/data/shape_check/tpcds_sf100/shape/query89.out index fecb0375185f1c..4032f2190b0d4d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query89.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query89.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (avg_monthly_sales = 0.0000)) and ((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) +------------filter(( not (tmp1.avg_monthly_sales = 0.0000)) and ((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------hashAgg[GLOBAL] @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Electronics', 'Jewelry', 'Shoes'),i_class IN ('athletic', 'portable', 'semi-precious')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'maternity', 'rock')]] and i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious')) +--------------------------------------filter(OR[AND[item.i_category IN ('Electronics', 'Jewelry', 'Shoes'),item.i_class IN ('athletic', 'portable', 'semi-precious')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'maternity', 'rock')]] and item.i_category IN ('Electronics', 'Jewelry', 'Men', 'Music', 'Shoes', 'Women') and item.i_class IN ('accessories', 'athletic', 'maternity', 'portable', 'rock', 'semi-precious')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 1999)) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query91.out b/regression-test/data/shape_check/tpcds_sf100/shape/query91.out index ea29f0373e8287..65cc169241fd61 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query91.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query91.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = customer.c_current_cdemo_sk)) otherCondition=() build RFs:RF2 c_current_cdemo_sk->cd_demo_sk --------------------------------PhysicalProject -----------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +----------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF2 --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((household_demographics.hd_demo_sk = customer.c_current_hdemo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->c_current_hdemo_sk @@ -31,7 +31,7 @@ PhysicalResultSink ------------------------------------------filter((customer_address.ca_gmt_offset = -6.00)) --------------------------------------------PhysicalOlapScan[customer_address] ------------------------------------PhysicalProject ---------------------------------------filter((hd_buy_potential like '1001-5000%')) +--------------------------------------filter((household_demographics.hd_buy_potential like '1001-5000%')) ----------------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query92.out b/regression-test/data/shape_check/tpcds_sf100/shape/query92.out index b02739c84db1ba..b69e7b837d186d 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query92.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query92.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +------------filter((cast(web_sales.ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query94.out b/regression-test/data/shape_check/tpcds_sf100/shape/query94.out index c5657bbc154bae..6be107b8333fc5 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query94.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query94.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF3 ws_order_number->ws_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF3 ws_order_number->ws_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF3 --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query95.out b/regression-test/data/shape_check/tpcds_sf100/shape/query95.out index cdc322aee59289..b239bf8a16c899 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query95.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number;RF1 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number;RF1 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF1 RF16 RF18 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query98.out b/regression-test/data/shape_check/tpcds_sf100/shape/query98.out index bc1091c292f35c..e334b81fa55857 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query98.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query98.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2002-06-19') and (date_dim.d_date >= '2002-05-20')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Music', 'Shoes', 'Sports')) +----------------------------filter(item.i_category IN ('Music', 'Shoes', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.out index a0294482523067..b7fea3a913beb2 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query13.out @@ -7,9 +7,9 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->ss_store_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->ca_address_sk ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) --------------------PhysicalOlapScan[customer_address] apply RFs: RF3 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4 ----------------------------PhysicalProject -------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W')) +------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'College', 'Primary') and customer_demographics.cd_marital_status IN ('D', 'M', 'W')) --------------------------------PhysicalOlapScan[customer_demographics] ------------------------PhysicalProject ---------------------------filter(hd_dep_count IN (1, 3)) +--------------------------filter(household_demographics.hd_dep_count IN (1, 3)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query19.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query19.out index 9b5a3b7e4afd77..019cea60019fbf 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query19.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) build RFs:RF4 c_current_addr_sk->ca_address_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) build RFs:RF4 c_current_addr_sk->ca_address_sk --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] apply RFs: RF4 --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query44.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query44.out index 6d4d5ed297e80c..8919be10a86fba 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query44.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query44.out @@ -36,7 +36,7 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF0 item_sk->i_item_sk @@ -66,6 +66,6 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query45.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query45.out index e4b5d2e59cf802..87f1424f7a7f72 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query45.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->ws_bill_customer_sk @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query56.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query56.out index d7d594193e5770..f99c21849a52de 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query56.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -78,6 +78,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query6.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query6.out index bab8096426d236..118ac1a7acc393 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query6.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query6.out @@ -36,7 +36,7 @@ PhysicalResultSink --------------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2002)) ----------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category +--------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item(i)] apply RFs: RF0 ----------------------------------hashAgg[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query68.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query68.out index f2db234dcb6611..a807c314e538a1 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query68.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address(current_addr)] apply RFs: RF5 ----------------PhysicalProject @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------filter(store.s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query8.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query8.out index e6b75f941a6c06..1e188ec784a6b4 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query8.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query8.out @@ -40,6 +40,6 @@ PhysicalResultSink ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +----------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ------------------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query91.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query91.out index b6b6254efc444d..1ceeaab6bc0f46 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query91.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query91.out @@ -28,10 +28,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1 ----------------------------------------PhysicalProject -------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) --------------------------------------------PhysicalOlapScan[customer_demographics] ------------------------------------PhysicalProject ---------------------------------------filter((hd_buy_potential like 'Unknown%')) +--------------------------------------filter((household_demographics.hd_buy_potential like 'Unknown%')) ----------------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query95.out b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query95.out index ad211327e44889..f0f5bc65c652da 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query95.out +++ b/regression-test/data/shape_check/tpcds_sf1000/bs_downgrade_shape/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF8 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query1.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query1.out index 055bc344b57acf..681b671e8b1d54 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query1.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query1.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalProject ----------------PhysicalOlapScan[customer] apply RFs: RF4 --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->ctr_store_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF2 diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query10.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query10.out index 88659d92226ac4..432db43a4a3dac 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query10.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query10.out @@ -35,7 +35,7 @@ PhysicalResultSink --------------------------------------filter((date_dim.d_moy <= 6) and (date_dim.d_moy >= 3) and (date_dim.d_year = 2001)) ----------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------filter(ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) +--------------------------------filter(ca.ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) ----------------------------------PhysicalOlapScan[customer_address(ca)] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->cs_sold_date_sk diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query12.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query12.out index 7d053dd110cfad..c2069590d687ae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query12.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query12.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2001-07-15') and (date_dim.d_date >= '2001-06-15')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Electronics', 'Men')) +----------------------------filter(item.i_category IN ('Books', 'Electronics', 'Men')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query13.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query13.out index a0294482523067..b7fea3a913beb2 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query13.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query13.out @@ -7,9 +7,9 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->ss_store_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->ca_address_sk ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) --------------------PhysicalOlapScan[customer_address] apply RFs: RF3 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4 ----------------------------PhysicalProject -------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W')) +------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'College', 'Primary') and customer_demographics.cd_marital_status IN ('D', 'M', 'W')) --------------------------------PhysicalOlapScan[customer_demographics] ------------------------PhysicalProject ---------------------------filter(hd_dep_count IN (1, 3)) +--------------------------filter(household_demographics.hd_dep_count IN (1, 3)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query15.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query15.out index 8755e0042ee5c5..6a70539628828d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query15.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query15.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),customer_address.ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_sold_date_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query16.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query16.out index f301947d87ea3b..a40f6ef4508e40 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query16.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query16.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[GLOBAL] ------------PhysicalProject ---------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF4 cs_order_number->cs_order_number +--------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF4 cs_order_number->cs_order_number ----------------PhysicalProject ------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF4 ----------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=() build RFs:RF3 cs_order_number->cr_order_number diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query17.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query17.out index e466a08541668f..0295dae51f145f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query17.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query17.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF8 RF9 RF10 ------------------------PhysicalProject ---------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------filter(d3.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------PhysicalOlapScan[date_dim(d3)] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->ss_item_sk;RF7 i_item_sk->sr_item_sk @@ -35,7 +35,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0 RF7 ------------------------------------PhysicalProject ---------------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------------------filter(d2.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------------------PhysicalOlapScan[date_dim(d2)] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query18.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query18.out index 25383a907832f8..b3caeec5b815e9 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query18.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query18.out @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->c_current_addr_sk --------------------------------------PhysicalProject -----------------------------------------filter(c_birth_month IN (1, 10, 11, 3, 4, 7)) +----------------------------------------filter(customer.c_birth_month IN (1, 10, 11, 3, 4, 7)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(ca_state IN ('AL', 'CA', 'GA', 'IN', 'MO', 'MT', 'TN')) +----------------------------------------filter(customer_address.ca_state IN ('AL', 'CA', 'GA', 'IN', 'MO', 'MT', 'TN')) ------------------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalProject ----------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query19.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query19.out index 2acb6c41511098..c59e137b008b1c 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query19.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 c_current_addr_sk->ca_address_sk ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query20.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query20.out index de6374e01e615c..3571ffd236e822 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query20.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query20.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2002-07-18') and (date_dim.d_date >= '2002-06-18')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Music', 'Sports')) +----------------------------filter(item.i_category IN ('Books', 'Music', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query21.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query21.out index f72c723c8eea92..daf78dcaa85b04 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query21.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query21.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) <= 1.5) and (if((inv_before > 0), (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) +--------filter(((cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)) <= 1.5) and (if((x.inv_before > 0), (cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query23.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query23.out index b1418479337d3b..08c611f83afa75 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query23.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query23.out @@ -14,7 +14,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------PhysicalProject -------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[item] @@ -26,7 +26,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------filter(( not ss_customer_sk IS NULL)) +------------------filter(( not store_sales.ss_customer_sk IS NULL)) --------------------PhysicalOlapScan[store_sales] ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecGather] @@ -38,10 +38,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ----------------------------PhysicalProject -------------------------------filter(( not ss_customer_sk IS NULL)) +------------------------------filter(( not store_sales.ss_customer_sk IS NULL)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ----------------------------PhysicalProject -------------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +------------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) --------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query24.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query24.out index 51e29c94825b12..a61711efe1ae4a 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query24.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query24.out @@ -17,7 +17,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------PhysicalProject --------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3 RF6 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF1 ca_address_sk->c_current_addr_sk +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (customer.c_birth_country = upper(customer_address.ca_country)))) build RFs:RF1 ca_address_sk->c_current_addr_sk ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer] apply RFs: RF1 ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query29.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query29.out index 0d0afdf7ac0b61..5cef5debe84e3b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query29.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query29.out @@ -38,6 +38,6 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] ------------------PhysicalProject ---------------------filter(d_year IN (1998, 1999, 2000)) +--------------------filter(d3.d_year IN (1998, 1999, 2000)) ----------------------PhysicalOlapScan[date_dim(d3)] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query30.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query30.out index d66a83eca1b438..f745e0e09b4543 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query30.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query30.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->ctr_customer_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query31.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query31.out index f127ca98cfe0e0..aa60158adf50cd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query31.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query31.out @@ -16,7 +16,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 --------------------PhysicalProject -----------------------filter((ss.d_year = 1999) and d_qoy IN (1, 2, 3)) +----------------------filter((ss.d_year = 1999) and ss.d_qoy IN (1, 2, 3)) ------------------------PhysicalOlapScan[date_dim] ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 ----------------------PhysicalProject -------------------------filter((ws.d_year = 1999) and d_qoy IN (1, 2, 3)) +------------------------filter((ws.d_year = 1999) and ws.d_qoy IN (1, 2, 3)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] @@ -45,7 +45,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalQuickSort[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF9 ca_county->ca_county +--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((ws2.web_sales > 0.00), (cast(ws3.web_sales as DECIMALV3(38, 8)) / ws2.web_sales), NULL) > if((ss2.store_sales > 0.00), (cast(ss3.store_sales as DECIMALV3(38, 8)) / ss2.store_sales), NULL))) build RFs:RF9 ca_county->ca_county ----------------PhysicalProject ------------------filter((ws3.d_qoy = 3) and (ws3.d_year = 1999)) --------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF9 @@ -54,7 +54,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------filter((ss3.d_qoy = 3) and (ss3.d_year = 1999)) ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8 ---------------------hashJoin[INNER_JOIN colocated] hashCondition=((ss1.ca_county = ws1.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF6 ca_county->ca_county;RF7 ca_county->ca_county +--------------------hashJoin[INNER_JOIN colocated] hashCondition=((ss1.ca_county = ws1.ca_county)) otherCondition=((if((ws1.web_sales > 0.00), (cast(ws2.web_sales as DECIMALV3(38, 8)) / ws1.web_sales), NULL) > if((ss1.store_sales > 0.00), (cast(ss2.store_sales as DECIMALV3(38, 8)) / ss1.store_sales), NULL))) build RFs:RF6 ca_county->ca_county;RF7 ca_county->ca_county ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ss1.ca_county = ss2.ca_county)) otherCondition=() build RFs:RF5 ca_county->ca_county ------------------------PhysicalProject --------------------------filter((ss1.d_qoy = 1) and (ss1.d_year = 1999)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query32.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query32.out index 7feac20dfc0b1d..d34db22ff7597f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query32.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query32.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +--------------filter((cast(catalog_sales.cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) ----------------PhysicalWindow ------------------PhysicalQuickSort[LOCAL_SORT] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query34.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query34.out index cf9b5c50e5a2f8..3eb97560e6fedd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query34.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query34.out @@ -24,9 +24,9 @@ PhysicalResultSink ----------------------------------filter((store.s_county = 'Williamson County')) ------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject -------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (2000, 2001, 2002)) +------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and date_dim.d_year IN (2000, 2001, 2002)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('0-500', '1001-5000')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('0-500', '1001-5000')) ----------------------------PhysicalOlapScan[household_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query37.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query37.out index f5a250d31657e4..1ba56407a2bf29 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query37.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query37.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 59.00) and (item.i_current_price >= 29.00) and i_manufact_id IN (705, 742, 777, 944)) +----------------------------filter((item.i_current_price <= 59.00) and (item.i_current_price >= 29.00) and item.i_manufact_id IN (705, 742, 777, 944)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2002-05-28') and (date_dim.d_date >= '2002-03-29')) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query39.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query39.out index 7786934fc2c37d..51254b45a1f265 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query39.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter(( not (mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) +------filter(( not (foo.mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) --------hashAgg[GLOBAL] ----------PhysicalProject ------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->inv_item_sk @@ -13,7 +13,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->inv_date_sk ----------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 RF2 ----------------------PhysicalProject -------------------------filter((date_dim.d_year = 2000) and d_moy IN (1, 2)) +------------------------filter((date_dim.d_year = 2000) and inv.d_moy IN (1, 2)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[warehouse] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query4.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query4.out index 7a759c8d508059..3f86cc6055b491 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query4.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query4.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF26 customer_id->c_customer_id;RF27 customer_id->c_customer_id;RF28 customer_id->c_customer_id;RF29 customer_id->c_customer_id;RF30 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_w_firstyear.year_total > 0.000000), (cast(t_w_secyear.year_total as DECIMALV3(38, 16)) / t_w_firstyear.year_total), NULL))) build RFs:RF26 customer_id->c_customer_id;RF27 customer_id->c_customer_id;RF28 customer_id->c_customer_id;RF29 customer_id->c_customer_id;RF30 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF21 customer_id->c_customer_id;RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF17 customer_id->c_customer_id;RF18 customer_id->c_customer_id;RF19 customer_id->c_customer_id +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.000000), (cast(t_s_secyear.year_total as DECIMALV3(38, 16)) / t_s_firstyear.year_total), NULL))) build RFs:RF17 customer_id->c_customer_id;RF18 customer_id->c_customer_id;RF19 customer_id->c_customer_id --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() build RFs:RF14 customer_id->c_customer_id;RF15 customer_id->c_customer_id ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF12 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query41.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query41.out index 3c19c29edabed1..da8ef93bfa0db5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query41.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query41.out @@ -18,6 +18,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'small'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('firebrick', 'maroon', 'sienna', 'smoke'),i_units IN ('Case', 'Cup', 'Each', 'Ounce'),OR[AND[i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce')],AND[i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each')]]],AND[i_size IN ('N/A', 'large'),i_color IN ('papaya', 'peach', 'powder', 'sky'),i_units IN ('Bundle', 'Carton', 'Dozen', 'Lb'),OR[AND[i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb')],AND[i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton')]]]]],AND[(item.i_category = 'Women'),i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('aquamarine', 'dark', 'forest', 'lime'),i_units IN ('Pallet', 'Pound', 'Tbl', 'Ton'),OR[AND[i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound')],AND[i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton')]]],AND[i_size IN ('extra large', 'petite'),i_color IN ('frosted', 'navy', 'plum', 'slate'),i_units IN ('Box', 'Bunch', 'Dram', 'Gross'),OR[AND[i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross')],AND[i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram')]]]]]] and i_category IN ('Men', 'Women')) +------------------------------filter(OR[AND[(item.i_category = 'Men'),item.i_size IN ('N/A', 'economy', 'large', 'small'),OR[AND[item.i_size IN ('economy', 'small'),item.i_color IN ('firebrick', 'maroon', 'sienna', 'smoke'),item.i_units IN ('Case', 'Cup', 'Each', 'Ounce'),OR[AND[item.i_color IN ('maroon', 'smoke'),item.i_units IN ('Case', 'Ounce')],AND[item.i_color IN ('firebrick', 'sienna'),item.i_units IN ('Cup', 'Each')]]],AND[item.i_size IN ('N/A', 'large'),item.i_color IN ('papaya', 'peach', 'powder', 'sky'),item.i_units IN ('Bundle', 'Carton', 'Dozen', 'Lb'),OR[AND[item.i_color IN ('powder', 'sky'),item.i_units IN ('Dozen', 'Lb')],AND[item.i_color IN ('papaya', 'peach'),item.i_units IN ('Bundle', 'Carton')]]]]],AND[(item.i_category = 'Women'),item.i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[item.i_size IN ('economy', 'small'),item.i_color IN ('aquamarine', 'dark', 'forest', 'lime'),item.i_units IN ('Pallet', 'Pound', 'Tbl', 'Ton'),OR[AND[item.i_color IN ('forest', 'lime'),item.i_units IN ('Pallet', 'Pound')],AND[item.i_color IN ('aquamarine', 'dark'),item.i_units IN ('Tbl', 'Ton')]]],AND[item.i_size IN ('extra large', 'petite'),item.i_color IN ('frosted', 'navy', 'plum', 'slate'),item.i_units IN ('Box', 'Bunch', 'Dram', 'Gross'),OR[AND[item.i_color IN ('navy', 'slate'),item.i_units IN ('Bunch', 'Gross')],AND[item.i_color IN ('frosted', 'plum'),item.i_units IN ('Box', 'Dram')]]]]]] and item.i_category IN ('Men', 'Women')) --------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query44.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query44.out index bb60e1d05e20b0..11516a282b0675 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query44.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query44.out @@ -33,7 +33,7 @@ PhysicalResultSink --------------------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------------------hashAgg[LOCAL] ------------------------------------------------PhysicalProject ---------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +--------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) ----------------------------------------------------PhysicalOlapScan[store_sales] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((i1.i_item_sk = ss1.ss_item_sk)) otherCondition=() build RFs:RF0 ss_item_sk->i_item_sk @@ -60,6 +60,6 @@ PhysicalResultSink --------------------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------------------hashAgg[LOCAL] ------------------------------------------------PhysicalProject ---------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +--------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) ----------------------------------------------------PhysicalOlapScan[store_sales] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query45.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query45.out index e4b5d2e59cf802..87f1424f7a7f72 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query45.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->ws_bill_customer_sk @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query46.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query46.out index 744f0531006f4f..05747df14fe249 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query46.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query46.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = ca_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk +----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = customer_address.ca_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk ------------PhysicalProject --------------hashJoin[INNER_JOIN shuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 ss_customer_sk->c_customer_sk ----------------PhysicalProject @@ -23,10 +23,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------filter(store.s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject -----------------------------------filter(d_dow IN (0, 6) and d_year IN (2000, 2001, 2002)) +----------------------------------filter(date_dim.d_dow IN (0, 6) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = 0)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query47.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query47.out index 36751e1448f03d..5781eb43dd3312 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query47.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query47.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and d_year IN (1999, 2000, 2001)) +----------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1999, 2000, 2001)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.i_brand = v1_lead.i_brand) and (v1.i_category = v1_lead.i_category) and (v1.rn = expr_(rn - 1)) and (v1.s_company_name = v1_lead.s_company_name) and (v1.s_store_name = v1_lead.s_store_name)) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 s_store_name->s_store_name;RF6 s_company_name->s_company_name;RF7 rn->(rn - 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF7 RF9 RF11 RF13 RF15 RF17 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8 RF10 RF12 RF14 RF16 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query48.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query48.out index 285f7009725792..fb613bda252a60 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query48.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query48.out @@ -9,17 +9,17 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[customer_address.ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[customer_address.ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->ss_cdemo_sk ------------------------PhysicalProject --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and customer_demographics.cd_marital_status IN ('D', 'M', 'S')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) +----------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) ------------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject ------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query53.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query53.out index 2f0af1334d0244..39fa03aaf32213 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query53.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query53.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197)) +----------------------------------filter(date_dim.d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out index 67b7645112458e..a0023a914cd339 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query54.out @@ -49,9 +49,9 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject -------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) <= d_month_seq+3) +------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) <= d_month_seq+3) --------------------------------PhysicalProject -----------------------------------NestedLoopJoin[INNER_JOIN](cast(d_month_seq as BIGINT) >= d_month_seq+1) +----------------------------------NestedLoopJoin[INNER_JOIN](cast(date_dim.d_month_seq as BIGINT) >= d_month_seq+1) ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalAssertNumRows diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query56.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query56.out index d7d594193e5770..f99c21849a52de 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query56.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -78,6 +78,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query57.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query57.out index b7cb9267a2b7eb..2db8f76c473c2e 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query57.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query57.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002)) +----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[call_center] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.cc_name = v1_lead.cc_name) and (v1.i_brand = v1_lead.i_brand) and (v1.i_category = v1_lead.i_category) and (v1.rn = expr_(rn - 1))) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 cc_name->cc_name;RF6 rn->(rn - 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF8 RF10 RF12 RF14 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7 RF9 RF11 RF13 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query58.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query58.out index 6f8d8a6f714895..dad201c2510f7c 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query58.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query58.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN colocated] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 i_item_id->i_item_id +------------hashJoin[INNER_JOIN colocated] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 i_item_id->i_item_id --------------hashAgg[GLOBAL] ----------------PhysicalDistribute[DistributionSpecHash] ------------------hashAgg[LOCAL] @@ -32,7 +32,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] apply RFs: RF13 --------------PhysicalProject -----------------hashJoin[INNER_JOIN colocated] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 i_item_id->i_item_id +----------------hashJoin[INNER_JOIN colocated] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 i_item_id->i_item_id ------------------hashAgg[GLOBAL] --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query6.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query6.out index f04bf1a0135439..1e006725eb32ce 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query6.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query6.out @@ -30,7 +30,7 @@ PhysicalResultSink ----------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2002)) ------------------------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF1 i_category->i_category +----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF1 i_category->i_category ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[item(i)] apply RFs: RF1 ------------------------------hashAgg[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query63.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query63.out index 6141f6c9936ad2..98a7877722b743 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query63.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query63.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233)) +----------------------------------filter(date_dim.d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query64.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query64.out index 373bb49922117b..6c5fdb95313b81 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query64.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query64.out @@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF13 p_promo_sk->ss_promo_sk --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF12 cd_demo_sk->c_current_cdemo_sk +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd1.cd_marital_status = cd2.cd_marital_status))) build RFs:RF12 cd_demo_sk->c_current_cdemo_sk ------------------------------------------------PhysicalProject --------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() build RFs:RF11 cd_demo_sk->ss_cdemo_sk ----------------------------------------------------PhysicalProject @@ -56,7 +56,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ------------------------------------------------------------------------------------------PhysicalProject --------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF24 ------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------filter(d_year IN (1999, 2000)) +--------------------------------------------------------------------------filter(d1.d_year IN (1999, 2000)) ----------------------------------------------------------------------------PhysicalOlapScan[date_dim(d1)] --------------------------------------------------------------------PhysicalProject ----------------------------------------------------------------------PhysicalOlapScan[store] @@ -85,7 +85,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------PhysicalProject ----------------------PhysicalOlapScan[income_band(ib2)] ----------------PhysicalProject -------------------filter((item.i_current_price <= 58.00) and (item.i_current_price >= 49.00) and i_color IN ('blush', 'lace', 'lawn', 'misty', 'orange', 'pink')) +------------------filter((item.i_current_price <= 58.00) and (item.i_current_price >= 49.00) and item.i_color IN ('blush', 'lace', 'lawn', 'misty', 'orange', 'pink')) --------------------PhysicalOlapScan[item] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query65.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query65.out index a073ab9a1701bf..a7d1233ad67cb0 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query65.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query65.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->ss_store_sk;RF4 s_store_sk->ss_store_sk --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF2 ss_store_sk->ss_store_sk +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(sc.revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF2 ss_store_sk->ss_store_sk ------------------------hashAgg[GLOBAL] --------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query66.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query66.out index 8da217bfbe4e89..14d6781ba2433c 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query66.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query66.out @@ -24,7 +24,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 2001)) @@ -48,7 +48,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5 RF6 RF7 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query68.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query68.out index 87a5a7fd9ea0db..6f5e6f00ad6e5e 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query68.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = ca_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = customer_address.ca_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address(current_addr)] apply RFs: RF5 ----------------PhysicalProject @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------filter(store.s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query69.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query69.out index aef57867a4289e..cf8b63691c9d44 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query69.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query69.out @@ -42,6 +42,6 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF0 --------------------------------PhysicalProject -----------------------------------filter(ca_state IN ('IL', 'ME', 'TX')) +----------------------------------filter(ca.ca_state IN ('IL', 'ME', 'TX')) ------------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query71.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query71.out index 02c38435318c49..635af091cc5bbf 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query71.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query71.out @@ -31,6 +31,6 @@ PhysicalResultSink --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2002)) ----------------------------PhysicalOlapScan[date_dim] --------------------PhysicalProject -----------------------filter(t_meal_time IN ('breakfast', 'dinner')) +----------------------filter(time_dim.t_meal_time IN ('breakfast', 'dinner')) ------------------------PhysicalOlapScan[time_dim] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query72.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query72.out index c0d8f9deafdd56..1d2ac153c8a102 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query72.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query72.out @@ -12,7 +12,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk)) otherCondition=() ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d_date, 5))) build RFs:RF9 d_date_sk->cs_ship_date_sk +------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d1.d_date, 5))) build RFs:RF9 d_date_sk->cs_ship_date_sk --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d1.d_week_seq = d2.d_week_seq) and (inventory.inv_date_sk = d2.d_date_sk)) otherCondition=() build RFs:RF7 d_week_seq->d_week_seq;RF8 d_date_sk->inv_date_sk ------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query73.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query73.out index c7ca5c1b783b57..d007c522ce6126 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query73.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query73.out @@ -21,12 +21,12 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter((store.s_county = 'Williamson County')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('1001-5000', '5001-10000')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('1001-5000', '5001-10000')) ----------------------------PhysicalOlapScan[household_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query75.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query75.out index e30d62d30ab824..0eda37accca422 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query75.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query75.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF6 ss_ticket_number->sr_ticket_number;RF7 ss_item_sk->sr_item_sk @@ -37,7 +37,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF10 ws_order_number->wr_order_number;RF11 ws_item_sk->wr_item_sk @@ -53,14 +53,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id +------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(curr_yr.sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(prev_yr.sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id --------------filter((curr_yr.d_year = 2002)) ----------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 RF13 RF14 RF15 --------------filter((prev_yr.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query76.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query76.out index a6280f1f5d2cc2..b182393366a7ae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query76.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query76.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->ss_item_sk --------------------------PhysicalProject -----------------------------filter(ss_customer_sk IS NULL) +----------------------------filter(store_sales.ss_customer_sk IS NULL) ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF3 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -23,13 +23,13 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] apply RFs: RF1 ------------------------PhysicalProject ---------------------------filter(ws_promo_sk IS NULL) +--------------------------filter(web_sales.ws_promo_sk IS NULL) ----------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 --------------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk --------------------------PhysicalProject -----------------------------filter(cs_bill_customer_sk IS NULL) +----------------------------filter(catalog_sales.cs_bill_customer_sk IS NULL) ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF5 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query78.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query78.out index ccd87748a08a1a..8bf0ea913683f2 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query78.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query78.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)]) +----------filter(OR[(coalesce(ws.ws_qty, 0) > 0),(coalesce(cs.cs_qty, 0) > 0)]) ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((catalog_sales.cs_bill_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=() --------------PhysicalProject ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((web_sales.ws_bill_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=() diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query79.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query79.out index ae09791f4eb56a..419ef5d8072c00 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query79.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query79.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dow = 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dow = 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 7),(household_demographics.hd_vehicle_count > -1)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query8.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query8.out index c168da88c79ec8..547e0c8dc10a28 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query8.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query8.out @@ -40,6 +40,6 @@ PhysicalResultSink --------------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 ------------------------------------------PhysicalProject ---------------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +--------------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ----------------------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query81.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query81.out index 18a63e0ff3b95b..a5e15fe37a2795 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query81.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query81.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->ctr_customer_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query82.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query82.out index e06e2ef7c59877..a6ab79b7c55bae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query82.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query82.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 88.00) and (item.i_current_price >= 58.00) and i_manufact_id IN (259, 485, 559, 580)) +----------------------------filter((item.i_current_price <= 88.00) and (item.i_current_price >= 58.00) and item.i_manufact_id IN (259, 485, 559, 580)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2001-03-14') and (date_dim.d_date >= '2001-01-13')) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query83.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query83.out index db167c011a3418..203eadd69aa4de 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query83.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query83.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF8 ----------------------------------------PhysicalProject -------------------------------------------filter(d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) +------------------------------------------filter(date_dim.d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) --------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[item] apply RFs: RF12 RF13 @@ -49,7 +49,7 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF4 ----------------------------------------PhysicalProject -------------------------------------------filter(d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) +------------------------------------------filter(date_dim.d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) --------------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[item] apply RFs: RF14 @@ -71,7 +71,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF0 ------------------------------------PhysicalProject ---------------------------------------filter(d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) +--------------------------------------filter(date_dim.d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) ----------------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query85.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query85.out index 073769756016cb..c9922690b393b6 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query85.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query85.out @@ -17,10 +17,10 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 wr_returning_cdemo_sk->cd_demo_sk --------------------------------PhysicalProject -----------------------------------filter(cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U')) +----------------------------------filter(cd2.cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd2.cd_marital_status IN ('D', 'S', 'U')) ------------------------------------PhysicalOlapScan[customer_demographics(cd2)] apply RFs: RF4 RF6 RF7 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->wr_refunded_addr_sk +----------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[customer_address.ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[customer_address.ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->wr_refunded_addr_sk ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->wr_item_sk;RF2 ws_order_number->wr_order_number ----------------------------------------PhysicalProject @@ -34,10 +34,10 @@ PhysicalResultSink ----------------------------------------------filter((date_dim.d_year = 1998)) ------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV')) +--------------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV')) ----------------------------------------PhysicalOlapScan[customer_address] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U')) +------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd1.cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd1.cd_marital_status IN ('D', 'S', 'U')) --------------------------------PhysicalOlapScan[customer_demographics(cd1)] ------------------------PhysicalProject --------------------------PhysicalOlapScan[web_page] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query88.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query88.out index 3727b0265bea9d..acf802da3f4a3b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query88.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query88.out @@ -23,7 +23,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -43,7 +43,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -63,7 +63,7 @@ PhysicalResultSink ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30)) ------------------------------------PhysicalOlapScan[time_dim] ----------------------------PhysicalProject -------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((store.s_store_name = 'ese')) @@ -83,7 +83,7 @@ PhysicalResultSink --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30)) ----------------------------------PhysicalOlapScan[time_dim] --------------------------PhysicalProject -----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------------PhysicalOlapScan[household_demographics] ----------------------PhysicalProject ------------------------filter((store.s_store_name = 'ese')) @@ -103,7 +103,7 @@ PhysicalResultSink ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30)) --------------------------------PhysicalOlapScan[time_dim] ------------------------PhysicalProject ---------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((store.s_store_name = 'ese')) @@ -123,7 +123,7 @@ PhysicalResultSink ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30)) ------------------------------PhysicalOlapScan[time_dim] ----------------------PhysicalProject -------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) --------------------------PhysicalOlapScan[household_demographics] ------------------PhysicalProject --------------------filter((store.s_store_name = 'ese')) @@ -143,7 +143,7 @@ PhysicalResultSink --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30)) ----------------------------PhysicalOlapScan[time_dim] --------------------PhysicalProject -----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------PhysicalOlapScan[household_demographics] ----------------PhysicalProject ------------------filter((store.s_store_name = 'ese')) @@ -163,7 +163,7 @@ PhysicalResultSink ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30)) --------------------------PhysicalOlapScan[time_dim] ------------------PhysicalProject ---------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------PhysicalOlapScan[household_demographics] --------------PhysicalProject ----------------filter((store.s_store_name = 'ese')) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query89.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query89.out index af729e84567695..e2ab2c93469038 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query89.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query89.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (avg_monthly_sales = 0.0000)) and ((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) +------------filter(( not (tmp1.avg_monthly_sales = 0.0000)) and ((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------hashAgg[GLOBAL] @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('audio', 'history', 'school-uniforms')],AND[i_category IN ('Men', 'Shoes', 'Sports'),i_class IN ('pants', 'tennis', 'womens')]] and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('audio', 'history', 'school-uniforms')],AND[item.i_category IN ('Men', 'Shoes', 'Sports'),item.i_class IN ('pants', 'tennis', 'womens')]] and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and item.i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query91.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query91.out index b6b6254efc444d..1ceeaab6bc0f46 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query91.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query91.out @@ -28,10 +28,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1 ----------------------------------------PhysicalProject -------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) --------------------------------------------PhysicalOlapScan[customer_demographics] ------------------------------------PhysicalProject ---------------------------------------filter((hd_buy_potential like 'Unknown%')) +--------------------------------------filter((household_demographics.hd_buy_potential like 'Unknown%')) ----------------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query92.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query92.out index 0f325df9d725e5..cd7b8a423bbd93 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query92.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query92.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +------------filter((cast(web_sales.ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query94.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query94.out index 18730d31c0c60e..2aa3ca40f33f8d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query94.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query94.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[GLOBAL] ------------PhysicalProject ---------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF4 ws_order_number->ws_order_number +--------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF4 ws_order_number->ws_order_number ----------------PhysicalProject ------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF4 ----------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=() build RFs:RF3 ws_order_number->wr_order_number diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query95.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query95.out index 4ea4a6222b430c..96877476635e5f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query95.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF8 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query97.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query97.out index 3f065cdc7a8c9b..d4cd9f969812ba 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query97.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query97.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ss_sold_date_sk --------------------------PhysicalProject -----------------------------filter(( not ss_sold_date_sk IS NULL)) +----------------------------filter(( not store_sales.ss_sold_date_sk IS NULL)) ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 --------------------------PhysicalProject ----------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199)) @@ -25,7 +25,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->cs_sold_date_sk --------------------------PhysicalProject -----------------------------filter(( not cs_sold_date_sk IS NULL)) +----------------------------filter(( not catalog_sales.cs_sold_date_sk IS NULL)) ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 --------------------------PhysicalProject ----------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query98.out b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query98.out index 4fd2b8c9253d66..b526a834c318bc 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/dphyper/query98.out +++ b/regression-test/data/shape_check/tpcds_sf1000/dphyper/query98.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '1999-03-07') and (date_dim.d_date >= '1999-02-05')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Jewelry', 'Men', 'Sports')) +----------------------------filter(item.i_category IN ('Jewelry', 'Men', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/eliminate_empty/query10_empty.out b/regression-test/data/shape_check/tpcds_sf1000/eliminate_empty/query10_empty.out index 3a6e62986761b7..cb5b61bb0787dd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/eliminate_empty/query10_empty.out +++ b/regression-test/data/shape_check/tpcds_sf1000/eliminate_empty/query10_empty.out @@ -42,6 +42,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) +------------------------------------filter(ca.ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) --------------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query1.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query1.out index 656e603c8b88ad..f33e20fd1aef19 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query1.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query1.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalProject ----------------PhysicalOlapScan[customer] apply RFs: RF4 --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->ctr_store_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF2 diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query10.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query10.out index 42e432aba9f3a8..b7e83f700f88d5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query10.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query10.out @@ -21,7 +21,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF3 RF5 ----------------------------------PhysicalProject -------------------------------------filter(ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) +------------------------------------filter(ca.ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) --------------------------------------PhysicalOlapScan[customer_address(ca)] --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query11.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query11.out index b6bb06a5da0ced..ab7bd6aa50fff2 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query11.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query11.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000) > if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000))) build RFs:RF13 customer_id->c_customer_id +----------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(38, 8)) / t_w_firstyear.year_total), 0.000000) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(38, 8)) / t_s_firstyear.year_total), 0.000000))) build RFs:RF13 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF12 c_customer_sk->ws_bill_customer_sk;RF14 customer_id->c_customer_id;RF15 customer_id->c_customer_id;RF16 customer_id->c_customer_id ----------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query12.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query12.out index dec39467c74612..f84c0c0bfcafad 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query12.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query12.out @@ -21,7 +21,7 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2001-07-15') and (date_dim.d_date >= '2001-06-15')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Electronics', 'Men')) +----------------------------filter(item.i_category IN ('Books', 'Electronics', 'Men')) ------------------------------PhysicalOlapScan[item] Hint log: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query13.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query13.out index 76fede37967b05..2654c78283b24f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query13.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query13.out @@ -9,25 +9,25 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF3 ss_cdemo_sk->cd_demo_sk ----------------PhysicalProject -------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W')) +------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'College', 'Primary') and customer_demographics.cd_marital_status IN ('D', 'M', 'W')) --------------------PhysicalOlapScan[customer_demographics] apply RFs: RF3 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->ss_hdemo_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ss_sold_date_sk ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->ss_addr_sk +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF0 ca_address_sk->ss_addr_sk ----------------------------PhysicalProject ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4 ----------------------------PhysicalProject -------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) +------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) --------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------filter((date_dim.d_year = 2001)) ----------------------------PhysicalOlapScan[date_dim] --------------------PhysicalProject -----------------------filter(hd_dep_count IN (1, 3)) +----------------------filter(household_demographics.hd_dep_count IN (1, 3)) ------------------------PhysicalOlapScan[household_demographics] ------------PhysicalProject --------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query15.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query15.out index 46573dae64ee04..670c5b23422058 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query15.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query15.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),customer_address.ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_sold_date_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query16.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query16.out index 5bb93fcc491558..34b03c22237790 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query16.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query16.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF4 cs_order_number->cs_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF4 cs_order_number->cs_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF4 --------------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=() build RFs:RF3 cs_order_number->cr_order_number diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query17.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query17.out index 8c64b2d40eed6b..af2ddca1552c3f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query17.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query17.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF8 RF9 RF10 ------------------------PhysicalProject ---------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------filter(d3.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------PhysicalOlapScan[date_dim(d3)] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->ss_item_sk;RF7 i_item_sk->sr_item_sk @@ -35,7 +35,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0 RF7 ------------------------------------PhysicalProject ---------------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------------------filter(d2.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------------------PhysicalOlapScan[date_dim(d2)] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query18.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query18.out index 79849ee172fe36..500ea6c01716a4 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query18.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query18.out @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->c_current_addr_sk --------------------------------------PhysicalProject -----------------------------------------filter(c_birth_month IN (1, 10, 11, 3, 4, 7)) +----------------------------------------filter(customer.c_birth_month IN (1, 10, 11, 3, 4, 7)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(ca_state IN ('AL', 'CA', 'GA', 'IN', 'MO', 'MT', 'TN')) +----------------------------------------filter(customer_address.ca_state IN ('AL', 'CA', 'GA', 'IN', 'MO', 'MT', 'TN')) ------------------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalProject ----------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query19.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query19.out index d858d5f3b02191..0fac5dd769c48d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query19.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 c_current_addr_sk->ca_address_sk ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query20.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query20.out index db4fae5d59f2ad..6bc5dbf3ab362d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query20.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query20.out @@ -21,7 +21,7 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2002-07-18') and (date_dim.d_date >= '2002-06-18')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Music', 'Sports')) +----------------------------filter(item.i_category IN ('Books', 'Music', 'Sports')) ------------------------------PhysicalOlapScan[item] Hint log: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query21.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query21.out index 7df4cfe9eebd7d..590af5fc736e1c 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query21.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query21.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) <= 1.5) and (if((inv_before > 0), (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) +--------filter(((cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)) <= 1.5) and (if((x.inv_before > 0), (cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query23.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query23.out index 9aab429718b123..f60df195058746 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query23.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query23.out @@ -14,7 +14,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------PhysicalProject -------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[item] @@ -27,7 +27,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------filter(( not ss_customer_sk IS NULL)) +--------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------PhysicalOlapScan[store_sales] ----------PhysicalProject ------------hashAgg[GLOBAL] @@ -40,10 +40,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ------------------------------PhysicalProject ---------------------------------filter(( not ss_customer_sk IS NULL)) +--------------------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ------------------------------PhysicalProject ---------------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query24.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query24.out index 8b6c616e026f00..dbf9afec2a88b5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query24.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query24.out @@ -20,7 +20,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------filter((store.s_market_id = 5)) --------------------------------PhysicalOlapScan[store] apply RFs: RF3 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 ca_address_sk->c_current_addr_sk +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (customer.c_birth_country = upper(customer_address.ca_country)))) build RFs:RF0 ca_address_sk->c_current_addr_sk ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query29.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query29.out index c66c256d346280..ce36f76ac272a8 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query29.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query29.out @@ -38,7 +38,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store] ------------------PhysicalProject ---------------------filter(d_year IN (1998, 1999, 2000)) +--------------------filter(d3.d_year IN (1998, 1999, 2000)) ----------------------PhysicalOlapScan[date_dim(d3)] Hint log: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query30.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query30.out index d875dc1cfe1ff2..4d9c6c18e61edf 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query30.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query30.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->ctr_customer_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query31.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query31.out index 1431c04c9695fe..33eda4d0e015f2 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query31.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query31.out @@ -16,7 +16,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 --------------------PhysicalProject -----------------------filter((ss.d_year = 1999) and d_qoy IN (1, 2, 3)) +----------------------filter((ss.d_year = 1999) and ss.d_qoy IN (1, 2, 3)) ------------------------PhysicalOlapScan[date_dim] ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 ----------------------PhysicalProject -------------------------filter((ws.d_year = 1999) and d_qoy IN (1, 2, 3)) +------------------------filter((ws.d_year = 1999) and ws.d_qoy IN (1, 2, 3)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] @@ -45,7 +45,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalQuickSort[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF11 ca_county->ca_county +--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((ws2.web_sales > 0.00), (cast(ws3.web_sales as DECIMALV3(38, 8)) / ws2.web_sales), NULL) > if((ss2.store_sales > 0.00), (cast(ss3.store_sales as DECIMALV3(38, 8)) / ss2.store_sales), NULL))) build RFs:RF11 ca_county->ca_county ----------------PhysicalProject ------------------filter((ws3.d_qoy = 3) and (ws3.d_year = 1999)) --------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF11 @@ -55,7 +55,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------filter((ss3.d_qoy = 3) and (ss3.d_year = 1999)) ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF10 --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF7 ca_county->ca_county;RF8 ca_county->ca_county;RF9 ca_county->ca_county +----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((ws1.web_sales > 0.00), (cast(ws2.web_sales as DECIMALV3(38, 8)) / ws1.web_sales), NULL) > if((ss1.store_sales > 0.00), (cast(ss2.store_sales as DECIMALV3(38, 8)) / ss1.store_sales), NULL))) build RFs:RF7 ca_county->ca_county;RF8 ca_county->ca_county;RF9 ca_county->ca_county ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss1.ca_county = ws1.ca_county)) otherCondition=() build RFs:RF5 ca_county->ca_county;RF6 ca_county->ca_county --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ss1.ca_county = ss2.ca_county)) otherCondition=() build RFs:RF4 ca_county->ca_county ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query32.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query32.out index 210efac04e55db..eb4073277652b1 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query32.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query32.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +--------------filter((cast(catalog_sales.cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) ----------------PhysicalWindow ------------------PhysicalQuickSort[LOCAL_SORT] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query34.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query34.out index 41b6692bd7cb07..0b2bafacbac324 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query34.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query34.out @@ -25,10 +25,10 @@ PhysicalResultSink ------------------------------------filter((store.s_county = 'Williamson County')) --------------------------------------PhysicalOlapScan[store] ------------------------------PhysicalProject ---------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (2000, 2001, 2002)) +--------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and date_dim.d_year IN (2000, 2001, 2002)) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('0-500', '1001-5000')) +----------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('0-500', '1001-5000')) ------------------------------PhysicalOlapScan[household_demographics] Hint log: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query37.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query37.out index adadbea8b15f80..104e517690b600 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query37.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query37.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 59.00) and (item.i_current_price >= 29.00) and i_manufact_id IN (705, 742, 777, 944)) +----------------------------filter((item.i_current_price <= 59.00) and (item.i_current_price >= 29.00) and item.i_manufact_id IN (705, 742, 777, 944)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2002-05-28') and (date_dim.d_date >= '2002-03-29')) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query39.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query39.out index 5fb1f43c9fa8df..57239e1c5a66c1 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query39.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter(( not (mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) +------filter(( not (foo.mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) --------hashAgg[GLOBAL] ----------PhysicalProject ------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->inv_item_sk @@ -13,7 +13,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->inv_date_sk ----------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 RF2 ----------------------PhysicalProject -------------------------filter((date_dim.d_year = 2000) and d_moy IN (1, 2)) +------------------------filter((date_dim.d_year = 2000) and inv.d_moy IN (1, 2)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[warehouse] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query4.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query4.out index bd2779101907d6..6fed68051fead5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query4.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query4.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF26 customer_id->c_customer_id +----------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_w_firstyear.year_total > 0.000000), (cast(t_w_secyear.year_total as DECIMALV3(38, 16)) / t_w_firstyear.year_total), NULL))) build RFs:RF26 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF25 c_customer_sk->ws_bill_customer_sk;RF27 customer_id->c_customer_id;RF28 customer_id->c_customer_id;RF29 customer_id->c_customer_id;RF30 customer_id->c_customer_id;RF31 customer_id->c_customer_id ----------------PhysicalProject @@ -40,7 +40,7 @@ PhysicalResultSink --------------------PhysicalProject ----------------------PhysicalOlapScan[customer] apply RFs: RF19 RF31 ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id;RF15 customer_id->c_customer_id +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.000000), (cast(t_s_secyear.year_total as DECIMALV3(38, 16)) / t_s_firstyear.year_total), NULL))) build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id;RF15 customer_id->c_customer_id --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() build RFs:RF10 customer_id->c_customer_id;RF11 customer_id->c_customer_id ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query41.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query41.out index 3c19c29edabed1..da8ef93bfa0db5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query41.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query41.out @@ -18,6 +18,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'small'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('firebrick', 'maroon', 'sienna', 'smoke'),i_units IN ('Case', 'Cup', 'Each', 'Ounce'),OR[AND[i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce')],AND[i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each')]]],AND[i_size IN ('N/A', 'large'),i_color IN ('papaya', 'peach', 'powder', 'sky'),i_units IN ('Bundle', 'Carton', 'Dozen', 'Lb'),OR[AND[i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb')],AND[i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton')]]]]],AND[(item.i_category = 'Women'),i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('aquamarine', 'dark', 'forest', 'lime'),i_units IN ('Pallet', 'Pound', 'Tbl', 'Ton'),OR[AND[i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound')],AND[i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton')]]],AND[i_size IN ('extra large', 'petite'),i_color IN ('frosted', 'navy', 'plum', 'slate'),i_units IN ('Box', 'Bunch', 'Dram', 'Gross'),OR[AND[i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross')],AND[i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram')]]]]]] and i_category IN ('Men', 'Women')) +------------------------------filter(OR[AND[(item.i_category = 'Men'),item.i_size IN ('N/A', 'economy', 'large', 'small'),OR[AND[item.i_size IN ('economy', 'small'),item.i_color IN ('firebrick', 'maroon', 'sienna', 'smoke'),item.i_units IN ('Case', 'Cup', 'Each', 'Ounce'),OR[AND[item.i_color IN ('maroon', 'smoke'),item.i_units IN ('Case', 'Ounce')],AND[item.i_color IN ('firebrick', 'sienna'),item.i_units IN ('Cup', 'Each')]]],AND[item.i_size IN ('N/A', 'large'),item.i_color IN ('papaya', 'peach', 'powder', 'sky'),item.i_units IN ('Bundle', 'Carton', 'Dozen', 'Lb'),OR[AND[item.i_color IN ('powder', 'sky'),item.i_units IN ('Dozen', 'Lb')],AND[item.i_color IN ('papaya', 'peach'),item.i_units IN ('Bundle', 'Carton')]]]]],AND[(item.i_category = 'Women'),item.i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[item.i_size IN ('economy', 'small'),item.i_color IN ('aquamarine', 'dark', 'forest', 'lime'),item.i_units IN ('Pallet', 'Pound', 'Tbl', 'Ton'),OR[AND[item.i_color IN ('forest', 'lime'),item.i_units IN ('Pallet', 'Pound')],AND[item.i_color IN ('aquamarine', 'dark'),item.i_units IN ('Tbl', 'Ton')]]],AND[item.i_size IN ('extra large', 'petite'),item.i_color IN ('frosted', 'navy', 'plum', 'slate'),item.i_units IN ('Box', 'Bunch', 'Dram', 'Gross'),OR[AND[item.i_color IN ('navy', 'slate'),item.i_units IN ('Bunch', 'Gross')],AND[item.i_color IN ('frosted', 'plum'),item.i_units IN ('Box', 'Dram')]]]]]] and item.i_category IN ('Men', 'Women')) --------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query44.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query44.out index 839dbf74e28ff5..c211eba487e612 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query44.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query44.out @@ -36,7 +36,7 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF0 item_sk->i_item_sk @@ -66,7 +66,7 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] Hint log: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query45.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query45.out index e4b5d2e59cf802..87f1424f7a7f72 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query45.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->ws_bill_customer_sk @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query46.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query46.out index 6a3344f7ac0ab8..2cc9d5c99b5b40 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query46.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query46.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk +----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk ------------PhysicalProject --------------hashJoin[INNER_JOIN shuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 ss_customer_sk->c_customer_sk ----------------PhysicalProject @@ -23,10 +23,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------filter(store.s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject -----------------------------------filter(d_dow IN (0, 6) and d_year IN (2000, 2001, 2002)) +----------------------------------filter(date_dim.d_dow IN (0, 6) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = 0)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query47.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query47.out index 99e00fb7675130..57aad3db2c9390 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query47.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query47.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and d_year IN (1999, 2000, 2001)) +----------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1999, 2000, 2001)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1)) and (v1.s_company_name = v1_lag.s_company_name) and (v1.s_store_name = v1_lag.s_store_name)) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 s_store_name->s_store_name;RF6 s_company_name->s_company_name;RF7 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF7 RF9 RF11 RF13 RF15 RF17 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8 RF10 RF12 RF14 RF16 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query48.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query48.out index df896f9ae80fdc..aadea1f5d9e4e1 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query48.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query48.out @@ -9,17 +9,17 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[customer_address.ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[customer_address.ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->ss_cdemo_sk ------------------------PhysicalProject --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and customer_demographics.cd_marital_status IN ('D', 'M', 'S')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) +----------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) ------------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject ------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query53.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query53.out index 1c3da568fa55e9..d644b0feed6c00 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query53.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query53.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(((cast(abs((sum_sales - cast(avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) +----------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) ------------PhysicalWindow --------------PhysicalQuickSort[LOCAL_SORT] ----------------PhysicalDistribute[DistributionSpecHash] @@ -22,10 +22,10 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------------PhysicalProject -----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +----------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ------------------------------------------PhysicalOlapScan[item] ----------------------------------PhysicalProject -------------------------------------filter(d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197)) +------------------------------------filter(date_dim.d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197)) --------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query56.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query56.out index d7d594193e5770..f99c21849a52de 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query56.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -78,6 +78,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query57.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query57.out index 04baedd5b08e10..a0609d49e517cb 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query57.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query57.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002)) +----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[call_center] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.cc_name = v1_lag.cc_name) and (v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1))) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 cc_name->cc_name;RF6 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF8 RF10 RF12 RF14 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7 RF9 RF11 RF13 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query58.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query58.out index df068a3012394e..b50a770d73da36 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query58.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query58.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id +------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id --------------PhysicalProject ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] @@ -33,7 +33,7 @@ PhysicalResultSink ------------------------------------filter((date_dim.d_date = '2001-06-16')) --------------------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject -----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id +----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id ------------------PhysicalProject --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query6.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query6.out index bab8096426d236..118ac1a7acc393 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query6.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query6.out @@ -36,7 +36,7 @@ PhysicalResultSink --------------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2002)) ----------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category +--------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item(i)] apply RFs: RF0 ----------------------------------hashAgg[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query63.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query63.out index fb6889408974c2..d8a108018efc85 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query63.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query63.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) +----------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) ------------PhysicalWindow --------------PhysicalQuickSort[LOCAL_SORT] ----------------PhysicalDistribute[DistributionSpecHash] @@ -22,10 +22,10 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------------PhysicalProject -----------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +----------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ------------------------------------------PhysicalOlapScan[item] ----------------------------------PhysicalProject -------------------------------------filter(d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233)) +------------------------------------filter(date_dim.d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233)) --------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query64.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query64.out index 7138669d2648b2..e3d76639779d3a 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query64.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query64.out @@ -30,7 +30,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ------------------------------------------------------PhysicalProject --------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_shipto_date_sk = d3.d_date_sk)) otherCondition=() build RFs:RF7 d_date_sk->c_first_shipto_date_sk ----------------------------------------------------------PhysicalProject -------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF6 cd_demo_sk->ss_cdemo_sk +------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=(( not (cd1.cd_marital_status = cd2.cd_marital_status))) build RFs:RF6 cd_demo_sk->ss_cdemo_sk --------------------------------------------------------------PhysicalProject ----------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->ss_customer_sk;RF16 c_current_addr_sk->ca_address_sk ------------------------------------------------------------------PhysicalProject @@ -50,7 +50,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------------------------------PhysicalProject ------------------------------------------------------------PhysicalOlapScan[date_dim(d3)] ------------------------------------------------------PhysicalProject ---------------------------------------------------------filter((item.i_current_price <= 58.00) and (item.i_current_price >= 49.00) and i_color IN ('blush', 'lace', 'lawn', 'misty', 'orange', 'pink')) +--------------------------------------------------------filter((item.i_current_price <= 58.00) and (item.i_current_price >= 49.00) and item.i_color IN ('blush', 'lace', 'lawn', 'misty', 'orange', 'pink')) ----------------------------------------------------------PhysicalOlapScan[item] apply RFs: RF11 RF22 ----------------------------------------------------PhysicalProject ------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd1.hd_income_band_sk = ib1.ib_income_band_sk)) otherCondition=() build RFs:RF2 ib_income_band_sk->hd_income_band_sk @@ -69,7 +69,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[income_band(ib2)] ----------------------------PhysicalProject -------------------------------filter(d_year IN (1999, 2000)) +------------------------------filter(d1.d_year IN (1999, 2000)) --------------------------------PhysicalOlapScan[date_dim(d1)] ------------------------PhysicalProject --------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query65.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query65.out index 187fa05e904c91..3e4e70463de814 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query65.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query65.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(sc.revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->ss_store_sk --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query66.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query66.out index 8da217bfbe4e89..14d6781ba2433c 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query66.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query66.out @@ -24,7 +24,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 2001)) @@ -48,7 +48,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5 RF6 RF7 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query68.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query68.out index baa24a1870a114..654548dd3ba7ac 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query68.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address(current_addr)] apply RFs: RF5 ----------------PhysicalProject @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------filter(store.s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query69.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query69.out index dbe8792fcda21a..f326b6c0c4a9a5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query69.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query69.out @@ -42,6 +42,6 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF0 --------------------------------PhysicalProject -----------------------------------filter(ca_state IN ('IL', 'ME', 'TX')) +----------------------------------filter(ca.ca_state IN ('IL', 'ME', 'TX')) ------------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query71.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query71.out index 1c75400a145774..cf0097c343dcc4 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query71.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query71.out @@ -31,7 +31,7 @@ PhysicalResultSink --------------------------filter((item.i_manager_id = 1)) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(t_meal_time IN ('breakfast', 'dinner')) +----------------------filter(time_dim.t_meal_time IN ('breakfast', 'dinner')) ------------------------PhysicalOlapScan[time_dim] Hint log: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query73.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query73.out index 0fa67bef2b09a2..7d11a7eb2ff11e 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query73.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query73.out @@ -22,13 +22,13 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------PhysicalProject -------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (2000, 2001, 2002)) +------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (2000, 2001, 2002)) --------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------filter((store.s_county = 'Williamson County')) ----------------------------------PhysicalOlapScan[store] --------------------------PhysicalProject -----------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('1001-5000', '5001-10000')) +----------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('1001-5000', '5001-10000')) ------------------------------PhysicalOlapScan[household_demographics] Hint log: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query74.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query74.out index 006e6f9d279ada..a6c43d526ce1bd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query74.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query74.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(13, 8)) / year_total), NULL) > if((year_total > 0.00), (cast(year_total as DECIMALV3(13, 8)) / year_total), NULL))) build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id;RF15 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(13, 8)) / t_w_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(13, 8)) / t_s_firstyear.year_total), NULL))) build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id;RF15 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF10 customer_id->c_customer_id;RF11 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query75.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query75.out index e30d62d30ab824..0eda37accca422 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query75.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query75.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF6 ss_ticket_number->sr_ticket_number;RF7 ss_item_sk->sr_item_sk @@ -37,7 +37,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF10 ws_order_number->wr_order_number;RF11 ws_item_sk->wr_item_sk @@ -53,14 +53,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id +------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(curr_yr.sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(prev_yr.sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id --------------filter((curr_yr.d_year = 2002)) ----------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 RF13 RF14 RF15 --------------filter((prev_yr.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query76.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query76.out index a6280f1f5d2cc2..b182393366a7ae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query76.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query76.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->ss_item_sk --------------------------PhysicalProject -----------------------------filter(ss_customer_sk IS NULL) +----------------------------filter(store_sales.ss_customer_sk IS NULL) ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF3 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -23,13 +23,13 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] apply RFs: RF1 ------------------------PhysicalProject ---------------------------filter(ws_promo_sk IS NULL) +--------------------------filter(web_sales.ws_promo_sk IS NULL) ----------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 --------------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk --------------------------PhysicalProject -----------------------------filter(cs_bill_customer_sk IS NULL) +----------------------------filter(catalog_sales.cs_bill_customer_sk IS NULL) ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF5 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query78.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query78.out index 7f0059faa7191b..9280fcd8d2e4ad 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query78.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query78.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)]) +----------filter(OR[(coalesce(ws.ws_qty, 0) > 0),(coalesce(cs.cs_qty, 0) > 0)]) ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=() --------------PhysicalProject ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=() diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query79.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query79.out index c4769401dd451c..f1c5139ad6439d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query79.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query79.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dow = 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dow = 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 7),(household_demographics.hd_vehicle_count > -1)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query8.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query8.out index dc9f5fef74d2d6..ab2a9c358e50cf 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query8.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query8.out @@ -40,7 +40,7 @@ PhysicalResultSink ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +----------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ------------------------------------------PhysicalOlapScan[customer_address] Hint log: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query81.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query81.out index 90bffeecc7284d..c3c8ff7707c22c 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query81.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query81.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 ctr_customer_sk->c_customer_sk;RF5 ctr_state->ctr_state ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query82.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query82.out index 817bbef3966593..3161601f947b25 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query82.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query82.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 88.00) and (item.i_current_price >= 58.00) and i_manufact_id IN (259, 485, 559, 580)) +----------------------------filter((item.i_current_price <= 88.00) and (item.i_current_price >= 58.00) and item.i_manufact_id IN (259, 485, 559, 580)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2001-03-14') and (date_dim.d_date >= '2001-01-13')) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query85.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query85.out index b5f020c7235c4f..2939c0feb797ef 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query85.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query85.out @@ -17,12 +17,12 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF5 wr_returning_cdemo_sk->cd_demo_sk;RF6 cd_marital_status->cd_marital_status;RF7 cd_education_status->cd_education_status ----------------------------PhysicalProject -------------------------------filter(cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U')) +------------------------------filter(cd2.cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd2.cd_marital_status IN ('D', 'S', 'U')) --------------------------------PhysicalOlapScan[customer_demographics(cd2)] apply RFs: RF5 RF6 RF7 ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->ca_address_sk +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[customer_address.ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[customer_address.ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->ca_address_sk --------------------------------PhysicalProject -----------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV')) +----------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV')) ------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4 --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->wr_refunded_cdemo_sk @@ -39,7 +39,7 @@ PhysicalResultSink ----------------------------------------------filter((date_dim.d_year = 1998)) ------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U')) +--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd1.cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd1.cd_marital_status IN ('D', 'S', 'U')) ----------------------------------------PhysicalOlapScan[customer_demographics(cd1)] ------------------------PhysicalProject --------------------------PhysicalOlapScan[reason] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query88.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query88.out index 96f5829a30cc9c..93ea8862cb3387 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query88.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query88.out @@ -24,7 +24,7 @@ PhysicalResultSink --------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30)) ----------------------------------------PhysicalOlapScan[time_dim] --------------------------------PhysicalProject -----------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------------------PhysicalOlapScan[household_demographics] ----------------------------PhysicalProject ------------------------------filter((store.s_store_name = 'ese')) @@ -45,7 +45,7 @@ PhysicalResultSink --------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30)) ----------------------------------------PhysicalOlapScan[time_dim] --------------------------------PhysicalProject -----------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------------------PhysicalOlapScan[household_demographics] ----------------------------PhysicalProject ------------------------------filter((store.s_store_name = 'ese')) @@ -66,7 +66,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -87,7 +87,7 @@ PhysicalResultSink ----------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30)) ------------------------------------PhysicalOlapScan[time_dim] ----------------------------PhysicalProject -------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((store.s_store_name = 'ese')) @@ -108,7 +108,7 @@ PhysicalResultSink --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30)) ----------------------------------PhysicalOlapScan[time_dim] --------------------------PhysicalProject -----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------------PhysicalOlapScan[household_demographics] ----------------------PhysicalProject ------------------------filter((store.s_store_name = 'ese')) @@ -129,7 +129,7 @@ PhysicalResultSink ------------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30)) --------------------------------PhysicalOlapScan[time_dim] ------------------------PhysicalProject ---------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((store.s_store_name = 'ese')) @@ -150,7 +150,7 @@ PhysicalResultSink ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30)) ------------------------------PhysicalOlapScan[time_dim] ----------------------PhysicalProject -------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) --------------------------PhysicalOlapScan[household_demographics] ------------------PhysicalProject --------------------filter((store.s_store_name = 'ese')) @@ -171,7 +171,7 @@ PhysicalResultSink --------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30)) ----------------------------PhysicalOlapScan[time_dim] --------------------PhysicalProject -----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------PhysicalOlapScan[household_demographics] ----------------PhysicalProject ------------------filter((store.s_store_name = 'ese')) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query89.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query89.out index 17499b73cc5017..e4ce11f401d471 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query89.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query89.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (avg_monthly_sales = 0.0000)) and ((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) +------------filter(( not (tmp1.avg_monthly_sales = 0.0000)) and ((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------hashAgg[GLOBAL] @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('audio', 'history', 'school-uniforms')],AND[i_category IN ('Men', 'Shoes', 'Sports'),i_class IN ('pants', 'tennis', 'womens')]] and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('audio', 'history', 'school-uniforms')],AND[item.i_category IN ('Men', 'Shoes', 'Sports'),item.i_class IN ('pants', 'tennis', 'womens')]] and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and item.i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query91.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query91.out index 6912b18abfc7b0..b0d2e4db5eaceb 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query91.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query91.out @@ -28,10 +28,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1 ----------------------------------------PhysicalProject -------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) --------------------------------------------PhysicalOlapScan[customer_demographics] ------------------------------------PhysicalProject ---------------------------------------filter((hd_buy_potential like 'Unknown%')) +--------------------------------------filter((household_demographics.hd_buy_potential like 'Unknown%')) ----------------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query92.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query92.out index 8bce5e2902950d..06dba3613389c9 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query92.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query92.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +------------filter((cast(web_sales.ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query94.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query94.out index 0c4c963c0b8fd0..e9553b57d751f3 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query94.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query94.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF4 ws_order_number->ws_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF4 ws_order_number->ws_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF4 --------------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=() build RFs:RF3 ws_order_number->wr_order_number diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query95.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query95.out index ad211327e44889..f0f5bc65c652da 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query95.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF8 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query97.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query97.out index a439c3a4cd22fc..96d3fa52142234 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query97.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query97.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ss_sold_date_sk ----------------------------PhysicalProject -------------------------------filter(( not ss_sold_date_sk IS NULL)) +------------------------------filter(( not store_sales.ss_sold_date_sk IS NULL)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 ----------------------------PhysicalProject ------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199)) @@ -27,7 +27,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->cs_sold_date_sk ----------------------------PhysicalProject -------------------------------filter(( not cs_sold_date_sk IS NULL)) +------------------------------filter(( not catalog_sales.cs_sold_date_sk IS NULL)) --------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 ----------------------------PhysicalProject ------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query98.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query98.out index 40a5dd3c3d0778..a831c702f9fe4d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query98.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query98.out @@ -21,7 +21,7 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '1999-03-07') and (date_dim.d_date >= '1999-02-05')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Jewelry', 'Men', 'Sports')) +----------------------------filter(item.i_category IN ('Jewelry', 'Men', 'Sports')) ------------------------------PhysicalOlapScan[item] Hint log: diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query1.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query1.out index 055bc344b57acf..681b671e8b1d54 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query1.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query1.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalProject ----------------PhysicalOlapScan[customer] apply RFs: RF4 --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->ctr_store_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF2 diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query10.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query10.out index 3a6e62986761b7..cb5b61bb0787dd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query10.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query10.out @@ -42,6 +42,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) +------------------------------------filter(ca.ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) --------------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query11.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query11.out index 50ff88fd878c14..d155c28b7bd318 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query11.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query11.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000) > if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000))) build RFs:RF13 c_customer_sk->ws_bill_customer_sk +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss_customer_sk = customer.c_customer_sk)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(38, 8)) / t_w_firstyear.year_total), 0.000000) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(38, 8)) / t_s_firstyear.year_total), 0.000000))) build RFs:RF13 c_customer_sk->ws_bill_customer_sk ------------PhysicalProject --------------hashAgg[GLOBAL] ----------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query12.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query12.out index 7d053dd110cfad..c2069590d687ae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query12.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query12.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2001-07-15') and (date_dim.d_date >= '2001-06-15')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Electronics', 'Men')) +----------------------------filter(item.i_category IN ('Books', 'Electronics', 'Men')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query13.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query13.out index a0294482523067..b7fea3a913beb2 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query13.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query13.out @@ -7,9 +7,9 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->ss_store_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->ca_address_sk ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) --------------------PhysicalOlapScan[customer_address] apply RFs: RF3 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4 ----------------------------PhysicalProject -------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W')) +------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'College', 'Primary') and customer_demographics.cd_marital_status IN ('D', 'M', 'W')) --------------------------------PhysicalOlapScan[customer_demographics] ------------------------PhysicalProject ---------------------------filter(hd_dep_count IN (1, 3)) +--------------------------filter(household_demographics.hd_dep_count IN (1, 3)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query15.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query15.out index 8755e0042ee5c5..6a70539628828d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query15.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query15.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),customer_address.ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_sold_date_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query16.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query16.out index 15445276dfaa0b..62ab186dfacddd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query16.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query16.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF4 cs_order_number->cs_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF4 cs_order_number->cs_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF4 --------------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=() build RFs:RF3 cs_order_number->cr_order_number diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query17.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query17.out index efc4fef3827959..69c966784666d0 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query17.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF8 RF9 RF10 ------------------------PhysicalProject ---------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------filter(d3.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------PhysicalOlapScan[date_dim(d3)] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->ss_item_sk;RF7 i_item_sk->sr_item_sk @@ -35,7 +35,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0 RF7 ------------------------------------PhysicalProject ---------------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------------------filter(d2.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------------------PhysicalOlapScan[date_dim(d2)] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query18.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query18.out index 25383a907832f8..b3caeec5b815e9 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query18.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query18.out @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->c_current_addr_sk --------------------------------------PhysicalProject -----------------------------------------filter(c_birth_month IN (1, 10, 11, 3, 4, 7)) +----------------------------------------filter(customer.c_birth_month IN (1, 10, 11, 3, 4, 7)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(ca_state IN ('AL', 'CA', 'GA', 'IN', 'MO', 'MT', 'TN')) +----------------------------------------filter(customer_address.ca_state IN ('AL', 'CA', 'GA', 'IN', 'MO', 'MT', 'TN')) ------------------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalProject ----------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query19.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query19.out index 9b5a3b7e4afd77..019cea60019fbf 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query19.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) build RFs:RF4 c_current_addr_sk->ca_address_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) build RFs:RF4 c_current_addr_sk->ca_address_sk --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] apply RFs: RF4 --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query20.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query20.out index de6374e01e615c..3571ffd236e822 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query20.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query20.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2002-07-18') and (date_dim.d_date >= '2002-06-18')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Music', 'Sports')) +----------------------------filter(item.i_category IN ('Books', 'Music', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query21.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query21.out index f72c723c8eea92..daf78dcaa85b04 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query21.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query21.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) <= 1.5) and (if((inv_before > 0), (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) +--------filter(((cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)) <= 1.5) and (if((x.inv_before > 0), (cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query23.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query23.out index 9aab429718b123..f60df195058746 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query23.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query23.out @@ -14,7 +14,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------PhysicalProject -------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[item] @@ -27,7 +27,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------filter(( not ss_customer_sk IS NULL)) +--------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------PhysicalOlapScan[store_sales] ----------PhysicalProject ------------hashAgg[GLOBAL] @@ -40,10 +40,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ------------------------------PhysicalProject ---------------------------------filter(( not ss_customer_sk IS NULL)) +--------------------------------filter(( not store_sales.ss_customer_sk IS NULL)) ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ------------------------------PhysicalProject ---------------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query24.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query24.out index 1d824ea2449602..1477a1699a8805 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query24.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query24.out @@ -20,7 +20,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------filter((store.s_market_id = 5)) --------------------------------PhysicalOlapScan[store] apply RFs: RF2 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 ca_address_sk->c_current_addr_sk +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (customer.c_birth_country = upper(customer_address.ca_country)))) build RFs:RF0 ca_address_sk->c_current_addr_sk ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query29.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query29.out index 0d0afdf7ac0b61..5cef5debe84e3b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query29.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query29.out @@ -38,6 +38,6 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] ------------------PhysicalProject ---------------------filter(d_year IN (1998, 1999, 2000)) +--------------------filter(d3.d_year IN (1998, 1999, 2000)) ----------------------PhysicalOlapScan[date_dim(d3)] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query30.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query30.out index 12710855d5d85a..482ba2bdddc32d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query30.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query30.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query31.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query31.out index c2b35be1eb957a..bb267f4f222257 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query31.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query31.out @@ -16,7 +16,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 --------------------PhysicalProject -----------------------filter((ss.d_year = 1999) and d_qoy IN (1, 2, 3)) +----------------------filter((ss.d_year = 1999) and ss.d_qoy IN (1, 2, 3)) ------------------------PhysicalOlapScan[date_dim] ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 ----------------------PhysicalProject -------------------------filter((ws.d_year = 1999) and d_qoy IN (1, 2, 3)) +------------------------filter((ws.d_year = 1999) and ws.d_qoy IN (1, 2, 3)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] @@ -45,12 +45,12 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalQuickSort[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF10 ca_county->ca_county +--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((ws2.web_sales > 0.00), (cast(ws3.web_sales as DECIMALV3(38, 8)) / ws2.web_sales), NULL) > if((ss2.store_sales > 0.00), (cast(ss3.store_sales as DECIMALV3(38, 8)) / ss2.store_sales), NULL))) build RFs:RF10 ca_county->ca_county ----------------PhysicalProject ------------------filter((ws3.d_qoy = 3) and (ws3.d_year = 1999)) --------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF10 ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF9 ca_county->ca_county +------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((ws1.web_sales > 0.00), (cast(ws2.web_sales as DECIMALV3(38, 8)) / ws1.web_sales), NULL) > if((ss1.store_sales > 0.00), (cast(ss2.store_sales as DECIMALV3(38, 8)) / ss1.store_sales), NULL))) build RFs:RF9 ca_county->ca_county --------------------PhysicalProject ----------------------filter((ws2.d_qoy = 2) and (ws2.d_year = 1999)) ------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF9 diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query32.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query32.out index 7feac20dfc0b1d..d34db22ff7597f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query32.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query32.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +--------------filter((cast(catalog_sales.cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) ----------------PhysicalWindow ------------------PhysicalQuickSort[LOCAL_SORT] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query34.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query34.out index cf9b5c50e5a2f8..3eb97560e6fedd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query34.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query34.out @@ -24,9 +24,9 @@ PhysicalResultSink ----------------------------------filter((store.s_county = 'Williamson County')) ------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject -------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (2000, 2001, 2002)) +------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and date_dim.d_year IN (2000, 2001, 2002)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('0-500', '1001-5000')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('0-500', '1001-5000')) ----------------------------PhysicalOlapScan[household_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query37.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query37.out index f5a250d31657e4..1ba56407a2bf29 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query37.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query37.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 59.00) and (item.i_current_price >= 29.00) and i_manufact_id IN (705, 742, 777, 944)) +----------------------------filter((item.i_current_price <= 59.00) and (item.i_current_price >= 29.00) and item.i_manufact_id IN (705, 742, 777, 944)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2002-05-28') and (date_dim.d_date >= '2002-03-29')) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query39.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query39.out index 7786934fc2c37d..51254b45a1f265 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query39.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter(( not (mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) +------filter(( not (foo.mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) --------hashAgg[GLOBAL] ----------PhysicalProject ------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->inv_item_sk @@ -13,7 +13,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->inv_date_sk ----------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 RF2 ----------------------PhysicalProject -------------------------filter((date_dim.d_year = 2000) and d_moy IN (1, 2)) +------------------------filter((date_dim.d_year = 2000) and inv.d_moy IN (1, 2)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[warehouse] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query4.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query4.out index 7a759c8d508059..3f86cc6055b491 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query4.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query4.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF26 customer_id->c_customer_id;RF27 customer_id->c_customer_id;RF28 customer_id->c_customer_id;RF29 customer_id->c_customer_id;RF30 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_w_firstyear.year_total > 0.000000), (cast(t_w_secyear.year_total as DECIMALV3(38, 16)) / t_w_firstyear.year_total), NULL))) build RFs:RF26 customer_id->c_customer_id;RF27 customer_id->c_customer_id;RF28 customer_id->c_customer_id;RF29 customer_id->c_customer_id;RF30 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF21 customer_id->c_customer_id;RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id;RF24 customer_id->c_customer_id ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF17 customer_id->c_customer_id;RF18 customer_id->c_customer_id;RF19 customer_id->c_customer_id +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.000000), (cast(t_s_secyear.year_total as DECIMALV3(38, 16)) / t_s_firstyear.year_total), NULL))) build RFs:RF17 customer_id->c_customer_id;RF18 customer_id->c_customer_id;RF19 customer_id->c_customer_id --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() build RFs:RF14 customer_id->c_customer_id;RF15 customer_id->c_customer_id ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF12 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query41.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query41.out index 3c19c29edabed1..da8ef93bfa0db5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query41.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query41.out @@ -18,6 +18,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'small'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('firebrick', 'maroon', 'sienna', 'smoke'),i_units IN ('Case', 'Cup', 'Each', 'Ounce'),OR[AND[i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce')],AND[i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each')]]],AND[i_size IN ('N/A', 'large'),i_color IN ('papaya', 'peach', 'powder', 'sky'),i_units IN ('Bundle', 'Carton', 'Dozen', 'Lb'),OR[AND[i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb')],AND[i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton')]]]]],AND[(item.i_category = 'Women'),i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('aquamarine', 'dark', 'forest', 'lime'),i_units IN ('Pallet', 'Pound', 'Tbl', 'Ton'),OR[AND[i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound')],AND[i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton')]]],AND[i_size IN ('extra large', 'petite'),i_color IN ('frosted', 'navy', 'plum', 'slate'),i_units IN ('Box', 'Bunch', 'Dram', 'Gross'),OR[AND[i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross')],AND[i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram')]]]]]] and i_category IN ('Men', 'Women')) +------------------------------filter(OR[AND[(item.i_category = 'Men'),item.i_size IN ('N/A', 'economy', 'large', 'small'),OR[AND[item.i_size IN ('economy', 'small'),item.i_color IN ('firebrick', 'maroon', 'sienna', 'smoke'),item.i_units IN ('Case', 'Cup', 'Each', 'Ounce'),OR[AND[item.i_color IN ('maroon', 'smoke'),item.i_units IN ('Case', 'Ounce')],AND[item.i_color IN ('firebrick', 'sienna'),item.i_units IN ('Cup', 'Each')]]],AND[item.i_size IN ('N/A', 'large'),item.i_color IN ('papaya', 'peach', 'powder', 'sky'),item.i_units IN ('Bundle', 'Carton', 'Dozen', 'Lb'),OR[AND[item.i_color IN ('powder', 'sky'),item.i_units IN ('Dozen', 'Lb')],AND[item.i_color IN ('papaya', 'peach'),item.i_units IN ('Bundle', 'Carton')]]]]],AND[(item.i_category = 'Women'),item.i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[item.i_size IN ('economy', 'small'),item.i_color IN ('aquamarine', 'dark', 'forest', 'lime'),item.i_units IN ('Pallet', 'Pound', 'Tbl', 'Ton'),OR[AND[item.i_color IN ('forest', 'lime'),item.i_units IN ('Pallet', 'Pound')],AND[item.i_color IN ('aquamarine', 'dark'),item.i_units IN ('Tbl', 'Ton')]]],AND[item.i_size IN ('extra large', 'petite'),item.i_color IN ('frosted', 'navy', 'plum', 'slate'),item.i_units IN ('Box', 'Bunch', 'Dram', 'Gross'),OR[AND[item.i_color IN ('navy', 'slate'),item.i_units IN ('Bunch', 'Gross')],AND[item.i_color IN ('frosted', 'plum'),item.i_units IN ('Box', 'Dram')]]]]]] and item.i_category IN ('Men', 'Women')) --------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query44.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query44.out index 6d4d5ed297e80c..8919be10a86fba 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query44.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query44.out @@ -36,7 +36,7 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF0 item_sk->i_item_sk @@ -66,6 +66,6 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query45.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query45.out index e4b5d2e59cf802..87f1424f7a7f72 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query45.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->ws_bill_customer_sk @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query46.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query46.out index 65512f9fb1356a..b2ec64f51f576a 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query46.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query46.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk +----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk ------------PhysicalProject --------------hashJoin[INNER_JOIN shuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 ss_customer_sk->c_customer_sk ----------------PhysicalProject @@ -23,10 +23,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------filter(store.s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject -----------------------------------filter(d_dow IN (0, 6) and d_year IN (2000, 2001, 2002)) +----------------------------------filter(date_dim.d_dow IN (0, 6) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = 0)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query47.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query47.out index 8ea0e362048748..a688a564e7278f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query47.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query47.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and d_year IN (1999, 2000, 2001)) +----------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1999, 2000, 2001)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1)) and (v1.s_company_name = v1_lag.s_company_name) and (v1.s_store_name = v1_lag.s_store_name)) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 s_store_name->s_store_name;RF6 s_company_name->s_company_name;RF7 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF7 RF9 RF11 RF13 RF15 RF17 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8 RF10 RF12 RF14 RF16 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query48.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query48.out index 285f7009725792..fb613bda252a60 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query48.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query48.out @@ -9,17 +9,17 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[customer_address.ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[customer_address.ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->ss_cdemo_sk ------------------------PhysicalProject --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and customer_demographics.cd_marital_status IN ('D', 'M', 'S')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) +----------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) ------------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject ------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query53.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query53.out index 2f0af1334d0244..39fa03aaf32213 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query53.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query53.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197)) +----------------------------------filter(date_dim.d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query56.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query56.out index d7d594193e5770..f99c21849a52de 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query56.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -78,6 +78,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query57.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query57.out index a9fd959fd9d037..ff725867be664d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query57.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query57.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002)) +----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[call_center] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.cc_name = v1_lag.cc_name) and (v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1))) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 cc_name->cc_name;RF6 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF8 RF10 RF12 RF14 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7 RF9 RF11 RF13 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query58.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query58.out index 27a0367b1b7044..2ccdc83269d76d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query58.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query58.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id +------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id --------------PhysicalProject ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] @@ -33,7 +33,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF13 --------------PhysicalProject -----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id +----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id ------------------PhysicalProject --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query6.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query6.out index bab8096426d236..118ac1a7acc393 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query6.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query6.out @@ -36,7 +36,7 @@ PhysicalResultSink --------------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2002)) ----------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category +--------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item(i)] apply RFs: RF0 ----------------------------------hashAgg[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query63.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query63.out index 6141f6c9936ad2..98a7877722b743 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query63.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query63.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233)) +----------------------------------filter(date_dim.d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query64.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query64.out deleted file mode 100644 index 7d5490b3d05c55..00000000000000 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query64.out +++ /dev/null @@ -1,102 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !ds_shape_64 -- -PhysicalCteAnchor ( cteId=CTEId#1 ) ---PhysicalCteProducer ( cteId=CTEId#1 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute[DistributionSpecHash] -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF19 i_item_sk->[cr_item_sk,cs_item_sk,sr_item_sk,ss_item_sk] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd2.hd_income_band_sk = ib2.ib_income_band_sk)) otherCondition=() build RFs:RF18 ib_income_band_sk->[hd_income_band_sk] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd1.hd_income_band_sk = ib1.ib_income_band_sk)) otherCondition=() build RFs:RF17 ib_income_band_sk->[hd_income_band_sk] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() build RFs:RF16 ca_address_sk->[c_current_addr_sk] -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = ad1.ca_address_sk)) otherCondition=() build RFs:RF15 ca_address_sk->[ss_addr_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_hdemo_sk = hd2.hd_demo_sk)) otherCondition=() build RFs:RF14 hd_demo_sk->[c_current_hdemo_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = hd1.hd_demo_sk)) otherCondition=() build RFs:RF13 hd_demo_sk->[ss_hdemo_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF12 p_promo_sk->[ss_promo_sk] ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF11 cd_demo_sk->[c_current_cdemo_sk] -------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() build RFs:RF10 cd_demo_sk->[ss_cdemo_sk] -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_shipto_date_sk = d3.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->[c_first_shipto_date_sk] ---------------------------------------------------------PhysicalProject -----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_sales_date_sk = d2.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[c_first_sales_date_sk] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ss_customer_sk] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF6 s_store_sk->[ss_store_sk] ---------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ss_sold_date_sk] -------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cs_ui.cs_item_sk)) otherCondition=() build RFs:RF4 cs_item_sk->[sr_item_sk,ss_item_sk] -----------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF2 sr_item_sk->[ss_item_sk];RF3 sr_ticket_number->[ss_ticket_number] ---------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3 RF4 RF5 RF6 RF7 RF10 RF12 RF13 RF15 RF19 ---------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF4 RF19 -----------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------filter((sale > (2 * refund))) ---------------------------------------------------------------------------------hashAgg[GLOBAL] -----------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------------------------------------------------------------hashAgg[LOCAL] ---------------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = catalog_returns.cr_item_sk) and (catalog_sales.cs_order_number = catalog_returns.cr_order_number)) otherCondition=() build RFs:RF0 cr_item_sk->[cs_item_sk];RF1 cr_order_number->[cs_order_number] -------------------------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF19 -------------------------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF19 -------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------filter(d_year IN (1999, 2000)) -----------------------------------------------------------------------------PhysicalOlapScan[date_dim(d1)] ---------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------PhysicalOlapScan[store] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF8 RF9 RF11 RF14 RF16 -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[date_dim(d2)] ---------------------------------------------------------PhysicalProject -----------------------------------------------------------PhysicalOlapScan[date_dim(d3)] -----------------------------------------------------PhysicalProject -------------------------------------------------------PhysicalOlapScan[customer_demographics(cd1)] -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer_demographics(cd2)] ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[promotion] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[household_demographics(hd1)] apply RFs: RF17 -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[household_demographics(hd2)] apply RFs: RF18 ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[customer_address(ad1)] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address(ad2)] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[income_band(ib1)] ---------------------PhysicalProject -----------------------PhysicalOlapScan[income_band(ib2)] -----------------PhysicalProject -------------------filter((item.i_current_price <= 58.00) and (item.i_current_price >= 49.00) and i_color IN ('blush', 'lace', 'lawn', 'misty', 'orange', 'pink')) ---------------------PhysicalOlapScan[item] ---PhysicalResultSink -----PhysicalQuickSort[MERGE_SORT] -------PhysicalDistribute[DistributionSpecGather] ---------PhysicalQuickSort[LOCAL_SORT] -----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((cs1.item_sk = cs2.item_sk) and (cs1.store_name = cs2.store_name) and (cs1.store_zip = cs2.store_zip)) otherCondition=((cs2.cnt <= cs1.cnt)) build RFs:RF20 item_sk->[item_sk];RF21 store_name->[store_name];RF22 store_zip->[store_zip] ---------------PhysicalProject -----------------filter((cs1.syear = 1999)) -------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF20 RF21 RF22 ---------------PhysicalProject -----------------filter((cs2.syear = 2000)) -------------------PhysicalCteConsumer ( cteId=CTEId#1 ) - diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query65.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query65.out index 9f7f2da0eba3aa..6ee1b83eb3ad3b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query65.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query65.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(sc.revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->ss_store_sk --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query66.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query66.out index 8da217bfbe4e89..14d6781ba2433c 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query66.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query66.out @@ -24,7 +24,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 2001)) @@ -48,7 +48,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5 RF6 RF7 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query68.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query68.out index f2db234dcb6611..a807c314e538a1 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query68.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address(current_addr)] apply RFs: RF5 ----------------PhysicalProject @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------filter(store.s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query69.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query69.out index dbe8792fcda21a..f326b6c0c4a9a5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query69.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query69.out @@ -42,6 +42,6 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF0 --------------------------------PhysicalProject -----------------------------------filter(ca_state IN ('IL', 'ME', 'TX')) +----------------------------------filter(ca.ca_state IN ('IL', 'ME', 'TX')) ------------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query71.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query71.out index 02c38435318c49..635af091cc5bbf 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query71.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query71.out @@ -31,6 +31,6 @@ PhysicalResultSink --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2002)) ----------------------------PhysicalOlapScan[date_dim] --------------------PhysicalProject -----------------------filter(t_meal_time IN ('breakfast', 'dinner')) +----------------------filter(time_dim.t_meal_time IN ('breakfast', 'dinner')) ------------------------PhysicalOlapScan[time_dim] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query72.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query72.out index bd3682190875fa..5283da3b319a60 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query72.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query72.out @@ -33,7 +33,7 @@ PhysicalResultSink --------------------------------------------------PhysicalProject ----------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 RF3 RF4 --------------------------------------------------PhysicalProject -----------------------------------------------------NestedLoopJoin[INNER_JOIN](d3.d_date > days_add(d_date, 5)) +----------------------------------------------------NestedLoopJoin[INNER_JOIN](d3.d_date > days_add(d1.d_date, 5)) ------------------------------------------------------PhysicalProject --------------------------------------------------------PhysicalOlapScan[date_dim(d3)] ------------------------------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query73.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query73.out index c7ca5c1b783b57..d007c522ce6126 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query73.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query73.out @@ -21,12 +21,12 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter((store.s_county = 'Williamson County')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('1001-5000', '5001-10000')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('1001-5000', '5001-10000')) ----------------------------PhysicalOlapScan[household_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query74.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query74.out index 006e6f9d279ada..a6c43d526ce1bd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query74.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query74.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(13, 8)) / year_total), NULL) > if((year_total > 0.00), (cast(year_total as DECIMALV3(13, 8)) / year_total), NULL))) build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id;RF15 customer_id->c_customer_id +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(13, 8)) / t_w_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(13, 8)) / t_s_firstyear.year_total), NULL))) build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id;RF15 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF10 customer_id->c_customer_id;RF11 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN shuffle] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query75.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query75.out index e30d62d30ab824..0eda37accca422 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query75.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query75.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF6 ss_ticket_number->sr_ticket_number;RF7 ss_item_sk->sr_item_sk @@ -37,7 +37,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF10 ws_order_number->wr_order_number;RF11 ws_item_sk->wr_item_sk @@ -53,14 +53,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id +------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(curr_yr.sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(prev_yr.sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id --------------filter((curr_yr.d_year = 2002)) ----------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 RF13 RF14 RF15 --------------filter((prev_yr.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query76.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query76.out index a6280f1f5d2cc2..b182393366a7ae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query76.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query76.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->ss_item_sk --------------------------PhysicalProject -----------------------------filter(ss_customer_sk IS NULL) +----------------------------filter(store_sales.ss_customer_sk IS NULL) ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF3 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -23,13 +23,13 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] apply RFs: RF1 ------------------------PhysicalProject ---------------------------filter(ws_promo_sk IS NULL) +--------------------------filter(web_sales.ws_promo_sk IS NULL) ----------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 --------------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk --------------------------PhysicalProject -----------------------------filter(cs_bill_customer_sk IS NULL) +----------------------------filter(catalog_sales.cs_bill_customer_sk IS NULL) ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF5 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query78.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query78.out index 0cb4bf14e4c5b8..aceea45035aad7 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query78.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query78.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)]) +----------filter(OR[(coalesce(ws.ws_qty, 0) > 0),(coalesce(cs.cs_qty, 0) > 0)]) ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=() --------------PhysicalProject ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=() diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query79.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query79.out index ae09791f4eb56a..419ef5d8072c00 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query79.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query79.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dow = 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dow = 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 7),(household_demographics.hd_vehicle_count > -1)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query8.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query8.out index e6b75f941a6c06..1e188ec784a6b4 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query8.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query8.out @@ -40,6 +40,6 @@ PhysicalResultSink ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +----------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ------------------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query81.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query81.out index df6f4e2f419e67..5aa31c018336bd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query81.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query81.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalDistribute[DistributionSpecGather] ------------PhysicalTopN[LOCAL_SORT] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF5 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF5 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query82.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query82.out index e06e2ef7c59877..a6ab79b7c55bae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query82.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query82.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 88.00) and (item.i_current_price >= 58.00) and i_manufact_id IN (259, 485, 559, 580)) +----------------------------filter((item.i_current_price <= 88.00) and (item.i_current_price >= 58.00) and item.i_manufact_id IN (259, 485, 559, 580)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2001-03-14') and (date_dim.d_date >= '2001-01-13')) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query83.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query83.out index e33fa4bc776525..e0ef2e6dbb9d56 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query83.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query83.out @@ -28,7 +28,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF8 ------------------------------------------PhysicalProject ---------------------------------------------filter(d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) +--------------------------------------------filter(date_dim.d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) ----------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[item] apply RFs: RF12 RF13 @@ -51,7 +51,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF4 ------------------------------------------PhysicalProject ---------------------------------------------filter(d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) +--------------------------------------------filter(date_dim.d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) ----------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[item] apply RFs: RF14 @@ -74,7 +74,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) +----------------------------------------filter(date_dim.d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) ------------------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query85.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query85.out index dc606c9794bd3d..809f9e744278dc 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query85.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query85.out @@ -13,14 +13,14 @@ PhysicalResultSink --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF6 wr_returning_cdemo_sk->cd_demo_sk;RF7 cd_marital_status->cd_marital_status;RF8 cd_education_status->cd_education_status ------------------------PhysicalProject ---------------------------filter(cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U')) +--------------------------filter(cd2.cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd2.cd_marital_status IN ('D', 'S', 'U')) ----------------------------PhysicalOlapScan[customer_demographics(cd2)] apply RFs: RF6 RF7 RF8 ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF5 wp_web_page_sk->ws_web_page_sk ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->ca_address_sk +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[customer_address.ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[customer_address.ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->ca_address_sk --------------------------------PhysicalProject -----------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV')) +----------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV')) ------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4 --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->wr_refunded_cdemo_sk @@ -37,7 +37,7 @@ PhysicalResultSink ----------------------------------------------filter((date_dim.d_year = 1998)) ------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U')) +--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd1.cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd1.cd_marital_status IN ('D', 'S', 'U')) ----------------------------------------PhysicalOlapScan[customer_demographics(cd1)] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_page] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query88.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query88.out index 3727b0265bea9d..acf802da3f4a3b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query88.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query88.out @@ -23,7 +23,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -43,7 +43,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -63,7 +63,7 @@ PhysicalResultSink ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30)) ------------------------------------PhysicalOlapScan[time_dim] ----------------------------PhysicalProject -------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((store.s_store_name = 'ese')) @@ -83,7 +83,7 @@ PhysicalResultSink --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30)) ----------------------------------PhysicalOlapScan[time_dim] --------------------------PhysicalProject -----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------------PhysicalOlapScan[household_demographics] ----------------------PhysicalProject ------------------------filter((store.s_store_name = 'ese')) @@ -103,7 +103,7 @@ PhysicalResultSink ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30)) --------------------------------PhysicalOlapScan[time_dim] ------------------------PhysicalProject ---------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((store.s_store_name = 'ese')) @@ -123,7 +123,7 @@ PhysicalResultSink ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30)) ------------------------------PhysicalOlapScan[time_dim] ----------------------PhysicalProject -------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) --------------------------PhysicalOlapScan[household_demographics] ------------------PhysicalProject --------------------filter((store.s_store_name = 'ese')) @@ -143,7 +143,7 @@ PhysicalResultSink --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30)) ----------------------------PhysicalOlapScan[time_dim] --------------------PhysicalProject -----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------PhysicalOlapScan[household_demographics] ----------------PhysicalProject ------------------filter((store.s_store_name = 'ese')) @@ -163,7 +163,7 @@ PhysicalResultSink ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30)) --------------------------PhysicalOlapScan[time_dim] ------------------PhysicalProject ---------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------PhysicalOlapScan[household_demographics] --------------PhysicalProject ----------------filter((store.s_store_name = 'ese')) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query89.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query89.out index af729e84567695..e2ab2c93469038 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query89.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query89.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (avg_monthly_sales = 0.0000)) and ((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) +------------filter(( not (tmp1.avg_monthly_sales = 0.0000)) and ((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------hashAgg[GLOBAL] @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('audio', 'history', 'school-uniforms')],AND[i_category IN ('Men', 'Shoes', 'Sports'),i_class IN ('pants', 'tennis', 'womens')]] and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('audio', 'history', 'school-uniforms')],AND[item.i_category IN ('Men', 'Shoes', 'Sports'),item.i_class IN ('pants', 'tennis', 'womens')]] and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and item.i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query91.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query91.out index b6b6254efc444d..1ceeaab6bc0f46 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query91.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query91.out @@ -28,10 +28,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1 ----------------------------------------PhysicalProject -------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) --------------------------------------------PhysicalOlapScan[customer_demographics] ------------------------------------PhysicalProject ---------------------------------------filter((hd_buy_potential like 'Unknown%')) +--------------------------------------filter((household_demographics.hd_buy_potential like 'Unknown%')) ----------------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query92.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query92.out index 0f325df9d725e5..cd7b8a423bbd93 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query92.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query92.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +------------filter((cast(web_sales.ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query94.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query94.out index 0c4c963c0b8fd0..e9553b57d751f3 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query94.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query94.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF4 ws_order_number->ws_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF4 ws_order_number->ws_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF4 --------------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=() build RFs:RF3 ws_order_number->wr_order_number diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query95.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query95.out index ad211327e44889..f0f5bc65c652da 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query95.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF8 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query97.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query97.out index ce92c6d6b072d0..302d502ca82dcb 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query97.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query97.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ss_sold_date_sk ----------------------------PhysicalProject -------------------------------filter(( not ss_sold_date_sk IS NULL)) +------------------------------filter(( not store_sales.ss_sold_date_sk IS NULL)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 ----------------------------PhysicalProject ------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199)) @@ -27,7 +27,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->cs_sold_date_sk ----------------------------PhysicalProject -------------------------------filter(( not cs_sold_date_sk IS NULL)) +------------------------------filter(( not catalog_sales.cs_sold_date_sk IS NULL)) --------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 ----------------------------PhysicalProject ------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199)) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query98.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query98.out index 4fd2b8c9253d66..b526a834c318bc 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query98.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query98.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '1999-03-07') and (date_dim.d_date >= '1999-02-05')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Jewelry', 'Men', 'Sports')) +----------------------------filter(item.i_category IN ('Jewelry', 'Men', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query1.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query1.out index 055bc344b57acf..681b671e8b1d54 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query1.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query1.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalProject ----------------PhysicalOlapScan[customer] apply RFs: RF4 --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_store_sk->ctr_store_sk;RF3 ctr_store_sk->s_store_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->ctr_store_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF1 RF2 diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query10.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query10.out index 3a6e62986761b7..cb5b61bb0787dd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query10.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query10.out @@ -42,6 +42,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) +------------------------------------filter(ca.ca_county IN ('Campbell County', 'Cleburne County', 'Escambia County', 'Fairfield County', 'Washtenaw County')) --------------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query11.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query11.out index 1ee38807411ce8..2845b323e1d1e5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query11.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query11.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000) > if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000))) build RFs:RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id +----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(38, 8)) / t_w_firstyear.year_total), 0.000000) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(38, 8)) / t_s_firstyear.year_total), 0.000000))) build RFs:RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF9 customer_id->c_customer_id;RF10 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id;RF11 customer_id->c_customer_id;RF15 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query12.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query12.out index 7d053dd110cfad..c2069590d687ae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query12.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query12.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2001-07-15') and (date_dim.d_date >= '2001-06-15')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Electronics', 'Men')) +----------------------------filter(item.i_category IN ('Books', 'Electronics', 'Men')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query13.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query13.out index a0294482523067..b7fea3a913beb2 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query13.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query13.out @@ -7,9 +7,9 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->ss_store_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IL', 'TN', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('ID', 'OH', 'WY'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IA', 'MS', 'SC'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ss_addr_sk->ca_address_sk ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('IA', 'ID', 'IL', 'MS', 'OH', 'SC', 'TN', 'TX', 'WY')) --------------------PhysicalOlapScan[customer_address] apply RFs: RF3 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4 ----------------------------PhysicalProject -------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and cd_education_status IN ('2 yr Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'M', 'W')) +------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'College')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'College', 'Primary') and customer_demographics.cd_marital_status IN ('D', 'M', 'W')) --------------------------------PhysicalOlapScan[customer_demographics] ------------------------PhysicalProject ---------------------------filter(hd_dep_count IN (1, 3)) +--------------------------filter(household_demographics.hd_dep_count IN (1, 3)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query15.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query15.out index 8755e0042ee5c5..6a70539628828d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query15.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query15.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),customer_address.ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF2 c_customer_sk->cs_bill_customer_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_sold_date_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query16.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query16.out index 15445276dfaa0b..62ab186dfacddd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query16.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query16.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF4 cs_order_number->cs_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF4 cs_order_number->cs_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF4 --------------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=() build RFs:RF3 cs_order_number->cr_order_number diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query17.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query17.out index efc4fef3827959..69c966784666d0 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query17.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF8 RF9 RF10 ------------------------PhysicalProject ---------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------filter(d3.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------PhysicalOlapScan[date_dim(d3)] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->ss_item_sk;RF7 i_item_sk->sr_item_sk @@ -35,7 +35,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0 RF7 ------------------------------------PhysicalProject ---------------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) +--------------------------------------filter(d2.d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3')) ----------------------------------------PhysicalOlapScan[date_dim(d2)] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query18.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query18.out index 25383a907832f8..b3caeec5b815e9 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query18.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query18.out @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->c_current_addr_sk --------------------------------------PhysicalProject -----------------------------------------filter(c_birth_month IN (1, 10, 11, 3, 4, 7)) +----------------------------------------filter(customer.c_birth_month IN (1, 10, 11, 3, 4, 7)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(ca_state IN ('AL', 'CA', 'GA', 'IN', 'MO', 'MT', 'TN')) +----------------------------------------filter(customer_address.ca_state IN ('AL', 'CA', 'GA', 'IN', 'MO', 'MT', 'TN')) ------------------------------------------PhysicalOlapScan[customer_address] --------------------------PhysicalProject ----------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query19.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query19.out index 9b5a3b7e4afd77..019cea60019fbf 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query19.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) build RFs:RF4 c_current_addr_sk->ca_address_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) build RFs:RF4 c_current_addr_sk->ca_address_sk --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] apply RFs: RF4 --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query20.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query20.out index de6374e01e615c..3571ffd236e822 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query20.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query20.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '2002-07-18') and (date_dim.d_date >= '2002-06-18')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Books', 'Music', 'Sports')) +----------------------------filter(item.i_category IN ('Books', 'Music', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query21.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query21.out index f72c723c8eea92..daf78dcaa85b04 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query21.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query21.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) <= 1.5) and (if((inv_before > 0), (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) +--------filter(((cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)) <= 1.5) and (if((x.inv_before > 0), (cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query23.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query23.out index 9da522aabd0ef6..51bc9e3b968d1c 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query23.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query23.out @@ -14,7 +14,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------PhysicalProject -------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[item] @@ -51,7 +51,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[customer] --------------------------PhysicalProject -----------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +----------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query24.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query24.out index 1d824ea2449602..1477a1699a8805 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query24.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query24.out @@ -20,7 +20,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------filter((store.s_market_id = 5)) --------------------------------PhysicalOlapScan[store] apply RFs: RF2 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 ca_address_sk->c_current_addr_sk +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (customer.c_birth_country = upper(customer_address.ca_country)))) build RFs:RF0 ca_address_sk->c_current_addr_sk ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query29.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query29.out index 0d0afdf7ac0b61..5cef5debe84e3b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query29.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query29.out @@ -38,6 +38,6 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] ------------------PhysicalProject ---------------------filter(d_year IN (1998, 1999, 2000)) +--------------------filter(d3.d_year IN (1998, 1999, 2000)) ----------------------PhysicalOlapScan[date_dim(d3)] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query30.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query30.out index adeabea2fef128..6b91175af78f55 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query30.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query30.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state +------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF4 ctr_state->ctr_state --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query31.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query31.out index c2b35be1eb957a..bb267f4f222257 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query31.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query31.out @@ -16,7 +16,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 --------------------PhysicalProject -----------------------filter((ss.d_year = 1999) and d_qoy IN (1, 2, 3)) +----------------------filter((ss.d_year = 1999) and ss.d_qoy IN (1, 2, 3)) ------------------------PhysicalOlapScan[date_dim] ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 ----------------------PhysicalProject -------------------------filter((ws.d_year = 1999) and d_qoy IN (1, 2, 3)) +------------------------filter((ws.d_year = 1999) and ws.d_qoy IN (1, 2, 3)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[customer_address] @@ -45,12 +45,12 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalQuickSort[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF10 ca_county->ca_county +--------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((ws2.web_sales > 0.00), (cast(ws3.web_sales as DECIMALV3(38, 8)) / ws2.web_sales), NULL) > if((ss2.store_sales > 0.00), (cast(ss3.store_sales as DECIMALV3(38, 8)) / ss2.store_sales), NULL))) build RFs:RF10 ca_county->ca_county ----------------PhysicalProject ------------------filter((ws3.d_qoy = 3) and (ws3.d_year = 1999)) --------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF10 ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF9 ca_county->ca_county +------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((ws1.web_sales > 0.00), (cast(ws2.web_sales as DECIMALV3(38, 8)) / ws1.web_sales), NULL) > if((ss1.store_sales > 0.00), (cast(ss2.store_sales as DECIMALV3(38, 8)) / ss1.store_sales), NULL))) build RFs:RF9 ca_county->ca_county --------------------PhysicalProject ----------------------filter((ws2.d_qoy = 2) and (ws2.d_year = 1999)) ------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF9 diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query32.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query32.out index 7feac20dfc0b1d..d34db22ff7597f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query32.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query32.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +--------------filter((cast(catalog_sales.cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) ----------------PhysicalWindow ------------------PhysicalQuickSort[LOCAL_SORT] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query34.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query34.out index cf9b5c50e5a2f8..3eb97560e6fedd 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query34.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query34.out @@ -24,9 +24,9 @@ PhysicalResultSink ----------------------------------filter((store.s_county = 'Williamson County')) ------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject -------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (2000, 2001, 2002)) +------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and date_dim.d_year IN (2000, 2001, 2002)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('0-500', '1001-5000')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('0-500', '1001-5000')) ----------------------------PhysicalOlapScan[household_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query37.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query37.out index f5a250d31657e4..1ba56407a2bf29 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query37.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query37.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 59.00) and (item.i_current_price >= 29.00) and i_manufact_id IN (705, 742, 777, 944)) +----------------------------filter((item.i_current_price <= 59.00) and (item.i_current_price >= 29.00) and item.i_manufact_id IN (705, 742, 777, 944)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2002-05-28') and (date_dim.d_date >= '2002-03-29')) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query39.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query39.out index 7786934fc2c37d..51254b45a1f265 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query39.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter(( not (mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) +------filter(( not (foo.mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) --------hashAgg[GLOBAL] ----------PhysicalProject ------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->inv_item_sk @@ -13,7 +13,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->inv_date_sk ----------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 RF2 ----------------------PhysicalProject -------------------------filter((date_dim.d_year = 2000) and d_moy IN (1, 2)) +------------------------filter((date_dim.d_year = 2000) and inv.d_moy IN (1, 2)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------PhysicalOlapScan[warehouse] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query4.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query4.out index 408dbe75fb0790..d6c9cde9fd1e42 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query4.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query4.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id;RF27 customer_id->c_customer_id;RF28 customer_id->c_customer_id;RF29 customer_id->c_customer_id +----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_w_firstyear.year_total > 0.000000), (cast(t_w_secyear.year_total as DECIMALV3(38, 16)) / t_w_firstyear.year_total), NULL))) build RFs:RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id;RF27 customer_id->c_customer_id;RF28 customer_id->c_customer_id;RF29 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF20 customer_id->c_customer_id;RF21 customer_id->c_customer_id;RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id ----------------PhysicalProject -------------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id;RF18 customer_id->c_customer_id +------------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.000000), (cast(t_s_secyear.year_total as DECIMALV3(38, 16)) / t_s_firstyear.year_total), NULL))) build RFs:RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id;RF18 customer_id->c_customer_id --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF12 customer_id->c_customer_id;RF15 customer_id->c_customer_id;RF19 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF30 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query41.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query41.out index 3c19c29edabed1..da8ef93bfa0db5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query41.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query41.out @@ -18,6 +18,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'economy', 'large', 'small'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('firebrick', 'maroon', 'sienna', 'smoke'),i_units IN ('Case', 'Cup', 'Each', 'Ounce'),OR[AND[i_color IN ('maroon', 'smoke'),i_units IN ('Case', 'Ounce')],AND[i_color IN ('firebrick', 'sienna'),i_units IN ('Cup', 'Each')]]],AND[i_size IN ('N/A', 'large'),i_color IN ('papaya', 'peach', 'powder', 'sky'),i_units IN ('Bundle', 'Carton', 'Dozen', 'Lb'),OR[AND[i_color IN ('powder', 'sky'),i_units IN ('Dozen', 'Lb')],AND[i_color IN ('papaya', 'peach'),i_units IN ('Bundle', 'Carton')]]]]],AND[(item.i_category = 'Women'),i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[i_size IN ('economy', 'small'),i_color IN ('aquamarine', 'dark', 'forest', 'lime'),i_units IN ('Pallet', 'Pound', 'Tbl', 'Ton'),OR[AND[i_color IN ('forest', 'lime'),i_units IN ('Pallet', 'Pound')],AND[i_color IN ('aquamarine', 'dark'),i_units IN ('Tbl', 'Ton')]]],AND[i_size IN ('extra large', 'petite'),i_color IN ('frosted', 'navy', 'plum', 'slate'),i_units IN ('Box', 'Bunch', 'Dram', 'Gross'),OR[AND[i_color IN ('navy', 'slate'),i_units IN ('Bunch', 'Gross')],AND[i_color IN ('frosted', 'plum'),i_units IN ('Box', 'Dram')]]]]]] and i_category IN ('Men', 'Women')) +------------------------------filter(OR[AND[(item.i_category = 'Men'),item.i_size IN ('N/A', 'economy', 'large', 'small'),OR[AND[item.i_size IN ('economy', 'small'),item.i_color IN ('firebrick', 'maroon', 'sienna', 'smoke'),item.i_units IN ('Case', 'Cup', 'Each', 'Ounce'),OR[AND[item.i_color IN ('maroon', 'smoke'),item.i_units IN ('Case', 'Ounce')],AND[item.i_color IN ('firebrick', 'sienna'),item.i_units IN ('Cup', 'Each')]]],AND[item.i_size IN ('N/A', 'large'),item.i_color IN ('papaya', 'peach', 'powder', 'sky'),item.i_units IN ('Bundle', 'Carton', 'Dozen', 'Lb'),OR[AND[item.i_color IN ('powder', 'sky'),item.i_units IN ('Dozen', 'Lb')],AND[item.i_color IN ('papaya', 'peach'),item.i_units IN ('Bundle', 'Carton')]]]]],AND[(item.i_category = 'Women'),item.i_size IN ('economy', 'extra large', 'petite', 'small'),OR[AND[item.i_size IN ('economy', 'small'),item.i_color IN ('aquamarine', 'dark', 'forest', 'lime'),item.i_units IN ('Pallet', 'Pound', 'Tbl', 'Ton'),OR[AND[item.i_color IN ('forest', 'lime'),item.i_units IN ('Pallet', 'Pound')],AND[item.i_color IN ('aquamarine', 'dark'),item.i_units IN ('Tbl', 'Ton')]]],AND[item.i_size IN ('extra large', 'petite'),item.i_color IN ('frosted', 'navy', 'plum', 'slate'),item.i_units IN ('Box', 'Bunch', 'Dram', 'Gross'),OR[AND[item.i_color IN ('navy', 'slate'),item.i_units IN ('Bunch', 'Gross')],AND[item.i_color IN ('frosted', 'plum'),item.i_units IN ('Box', 'Dram')]]]]]] and item.i_category IN ('Men', 'Women')) --------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query44.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query44.out index 6d4d5ed297e80c..8919be10a86fba 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query44.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query44.out @@ -36,7 +36,7 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF0 item_sk->i_item_sk @@ -66,6 +66,6 @@ PhysicalResultSink ------------------------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------------------------hashAgg[LOCAL] ----------------------------------------------------PhysicalProject -------------------------------------------------------filter((store_sales.ss_store_sk = 4) and ss_hdemo_sk IS NULL) +------------------------------------------------------filter((store_sales.ss_store_sk = 4) and store_sales.ss_hdemo_sk IS NULL) --------------------------------------------------------PhysicalOlapScan[store_sales] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query45.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query45.out index e4b5d2e59cf802..87f1424f7a7f72 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query45.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->ws_bill_customer_sk @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query46.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query46.out index 65512f9fb1356a..b2ec64f51f576a 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query46.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query46.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk +----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk ------------PhysicalProject --------------hashJoin[INNER_JOIN shuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 ss_customer_sk->c_customer_sk ----------------PhysicalProject @@ -23,10 +23,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------filter(store.s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject -----------------------------------filter(d_dow IN (0, 6) and d_year IN (2000, 2001, 2002)) +----------------------------------filter(date_dim.d_dow IN (0, 6) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 8),(household_demographics.hd_vehicle_count = 0)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query47.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query47.out index 8ea0e362048748..a688a564e7278f 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query47.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query47.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and d_year IN (1999, 2000, 2001)) +----------------------------------filter(OR[(date_dim.d_year = 2000),AND[(date_dim.d_year = 1999),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2001),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1999, 2000, 2001)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1)) and (v1.s_company_name = v1_lag.s_company_name) and (v1.s_store_name = v1_lag.s_store_name)) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 s_store_name->s_store_name;RF6 s_company_name->s_company_name;RF7 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF7 RF9 RF11 RF13 RF15 RF17 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2000)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF8 RF10 RF12 RF14 RF16 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query48.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query48.out index 285f7009725792..fb613bda252a60 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query48.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query48.out @@ -9,17 +9,17 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ss_sold_date_sk ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('ND', 'NY', 'SD'),(store_sales.ss_net_profit <= 2000.00)],AND[customer_address.ca_state IN ('GA', 'KS', 'MD'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[customer_address.ca_state IN ('CO', 'MN', 'NC'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF1 ca_address_sk->ss_addr_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF0 cd_demo_sk->ss_cdemo_sk ------------------------PhysicalProject --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and cd_marital_status IN ('D', 'M', 'S')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'S'),(customer_demographics.cd_education_status = 'Secondary')],AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = '2 yr Degree')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('2 yr Degree', 'Advanced Degree', 'Secondary') and customer_demographics.cd_marital_status IN ('D', 'M', 'S')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) +----------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) ------------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject ------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query53.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query53.out index 2f0af1334d0244..39fa03aaf32213 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query53.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query53.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197)) +----------------------------------filter(date_dim.d_month_seq IN (1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query56.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query56.out index d7d594193e5770..f99c21849a52de 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query56.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((customer_address.ca_gmt_offset = -6.00)) @@ -78,6 +78,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('orchid', 'pink', 'powder')) +------------------------------------filter(item.i_color IN ('orchid', 'pink', 'powder')) --------------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query57.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query57.out index a9fd959fd9d037..ff725867be664d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query57.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query57.out @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and d_year IN (2000, 2001, 2002)) +----------------------------------filter(OR[(date_dim.d_year = 2001),AND[(date_dim.d_year = 2000),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2002),(date_dim.d_moy = 1)]] and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[call_center] @@ -36,7 +36,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.cc_name = v1_lag.cc_name) and (v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1))) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 cc_name->cc_name;RF6 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF8 RF10 RF12 RF14 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF7 RF9 RF11 RF13 ----------------PhysicalProject ------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query58.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query58.out index 27a0367b1b7044..2ccdc83269d76d 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query58.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query58.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id +------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id --------------PhysicalProject ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] @@ -33,7 +33,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] apply RFs: RF13 --------------PhysicalProject -----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id +----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF8 item_id->i_item_id ------------------PhysicalProject --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query6.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query6.out index bab8096426d236..118ac1a7acc393 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query6.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query6.out @@ -36,7 +36,7 @@ PhysicalResultSink --------------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2002)) ----------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category +--------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF0 i_category->i_category ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item(i)] apply RFs: RF0 ----------------------------------hashAgg[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query63.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query63.out index 6141f6c9936ad2..98a7877722b743 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query63.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query63.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalDistribute[DistributionSpecHash] @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject -----------------------------------filter(d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233)) +----------------------------------filter(date_dim.d_month_seq IN (1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query64.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query64.out deleted file mode 100644 index 13e1bc08953a35..00000000000000 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query64.out +++ /dev/null @@ -1,102 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !ds_shape_64 -- -PhysicalCteAnchor ( cteId=CTEId#1 ) ---PhysicalCteProducer ( cteId=CTEId#1 ) -----PhysicalProject -------hashAgg[GLOBAL] ---------PhysicalDistribute[DistributionSpecHash] -----------hashAgg[LOCAL] -------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF19 i_item_sk->[cr_item_sk,cs_item_sk,sr_item_sk,ss_item_sk] -----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd2.hd_income_band_sk = ib2.ib_income_band_sk)) otherCondition=() build RFs:RF18 ib_income_band_sk->[hd_income_band_sk] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd1.hd_income_band_sk = ib1.ib_income_band_sk)) otherCondition=() build RFs:RF17 ib_income_band_sk->[hd_income_band_sk] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() build RFs:RF16 ca_address_sk->[c_current_addr_sk] -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = ad1.ca_address_sk)) otherCondition=() build RFs:RF15 ca_address_sk->[ss_addr_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_hdemo_sk = hd2.hd_demo_sk)) otherCondition=() build RFs:RF14 hd_demo_sk->[c_current_hdemo_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = hd1.hd_demo_sk)) otherCondition=() build RFs:RF13 hd_demo_sk->[ss_hdemo_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF12 p_promo_sk->[ss_promo_sk] ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF11 cd_demo_sk->[c_current_cdemo_sk] -------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() build RFs:RF10 cd_demo_sk->[ss_cdemo_sk] -----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_shipto_date_sk = d3.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->[c_first_shipto_date_sk] ---------------------------------------------------------PhysicalProject -----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_sales_date_sk = d2.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[c_first_sales_date_sk] -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ss_customer_sk] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF6 s_store_sk->[ss_store_sk] ---------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF4 sr_item_sk->[cr_item_sk,cs_item_sk,ss_item_sk];RF5 sr_ticket_number->[ss_ticket_number] -------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cs_ui.cs_item_sk)) otherCondition=() build RFs:RF3 cs_item_sk->[ss_item_sk] -----------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk] ---------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3 RF4 RF5 RF6 RF7 RF10 RF12 RF13 RF15 RF19 ---------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------filter(d_year IN (1999, 2000)) -------------------------------------------------------------------------------------PhysicalOlapScan[date_dim(d1)] -----------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------filter((sale > (2 * refund))) ---------------------------------------------------------------------------------hashAgg[GLOBAL] -----------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------------------------------------------------------------hashAgg[LOCAL] ---------------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = catalog_returns.cr_item_sk) and (catalog_sales.cs_order_number = catalog_returns.cr_order_number)) otherCondition=() build RFs:RF0 cr_item_sk->[cs_item_sk];RF1 cr_order_number->[cs_order_number] -------------------------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF4 RF19 -------------------------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF4 RF19 -------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF19 ---------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------PhysicalOlapScan[store] -----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF8 RF9 RF11 RF14 RF16 -------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[date_dim(d2)] ---------------------------------------------------------PhysicalProject -----------------------------------------------------------PhysicalOlapScan[date_dim(d3)] -----------------------------------------------------PhysicalProject -------------------------------------------------------PhysicalOlapScan[customer_demographics(cd1)] -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[customer_demographics(cd2)] ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[promotion] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[household_demographics(hd1)] apply RFs: RF17 -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[household_demographics(hd2)] apply RFs: RF18 ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[customer_address(ad1)] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address(ad2)] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[income_band(ib1)] ---------------------PhysicalProject -----------------------PhysicalOlapScan[income_band(ib2)] -----------------PhysicalProject -------------------filter((item.i_current_price <= 58.00) and (item.i_current_price >= 49.00) and i_color IN ('blush', 'lace', 'lawn', 'misty', 'orange', 'pink')) ---------------------PhysicalOlapScan[item] ---PhysicalResultSink -----PhysicalQuickSort[MERGE_SORT] -------PhysicalDistribute[DistributionSpecGather] ---------PhysicalQuickSort[LOCAL_SORT] -----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((cs1.item_sk = cs2.item_sk) and (cs1.store_name = cs2.store_name) and (cs1.store_zip = cs2.store_zip)) otherCondition=((cs2.cnt <= cs1.cnt)) build RFs:RF20 item_sk->[item_sk];RF21 store_name->[store_name];RF22 store_zip->[store_zip] ---------------PhysicalProject -----------------filter((cs1.syear = 1999)) -------------------PhysicalCteConsumer ( cteId=CTEId#1 ) apply RFs: RF20 RF21 RF22 ---------------PhysicalProject -----------------filter((cs2.syear = 2000)) -------------------PhysicalCteConsumer ( cteId=CTEId#1 ) - diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query65.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query65.out index 9f7f2da0eba3aa..6ee1b83eb3ad3b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query65.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query65.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(sc.revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->ss_store_sk --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query66.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query66.out index 8da217bfbe4e89..14d6781ba2433c 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query66.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query66.out @@ -24,7 +24,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 RF2 RF3 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 2001)) @@ -48,7 +48,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5 RF6 RF7 ------------------------------------------PhysicalProject ---------------------------------------------filter(sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) +--------------------------------------------filter(ship_mode.sm_carrier IN ('BOXBUNDLES', 'ORIENTAL')) ----------------------------------------------PhysicalOlapScan[ship_mode] --------------------------------------PhysicalProject ----------------------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query68.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query68.out index f2db234dcb6611..a807c314e538a1 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query68.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 c_current_addr_sk->ca_address_sk ----------------PhysicalProject ------------------PhysicalOlapScan[customer_address(current_addr)] apply RFs: RF5 ----------------PhysicalProject @@ -29,10 +29,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1998, 1999, 2000)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Fairview', 'Midway')) +--------------------------------------filter(store.s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count = 4)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query69.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query69.out index dbe8792fcda21a..f326b6c0c4a9a5 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query69.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query69.out @@ -42,6 +42,6 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF0 --------------------------------PhysicalProject -----------------------------------filter(ca_state IN ('IL', 'ME', 'TX')) +----------------------------------filter(ca.ca_state IN ('IL', 'ME', 'TX')) ------------------------------------PhysicalOlapScan[customer_address(ca)] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query71.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query71.out index 02c38435318c49..635af091cc5bbf 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query71.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query71.out @@ -31,6 +31,6 @@ PhysicalResultSink --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2002)) ----------------------------PhysicalOlapScan[date_dim] --------------------PhysicalProject -----------------------filter(t_meal_time IN ('breakfast', 'dinner')) +----------------------filter(time_dim.t_meal_time IN ('breakfast', 'dinner')) ------------------------PhysicalOlapScan[time_dim] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query72.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query72.out index bd3682190875fa..5283da3b319a60 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query72.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query72.out @@ -33,7 +33,7 @@ PhysicalResultSink --------------------------------------------------PhysicalProject ----------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 RF3 RF4 --------------------------------------------------PhysicalProject -----------------------------------------------------NestedLoopJoin[INNER_JOIN](d3.d_date > days_add(d_date, 5)) +----------------------------------------------------NestedLoopJoin[INNER_JOIN](d3.d_date > days_add(d1.d_date, 5)) ------------------------------------------------------PhysicalProject --------------------------------------------------------PhysicalOlapScan[date_dim(d3)] ------------------------------------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query73.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query73.out index c7ca5c1b783b57..d007c522ce6126 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query73.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query73.out @@ -21,12 +21,12 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter((store.s_county = 'Williamson County')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('1001-5000', '5001-10000')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('1001-5000', '5001-10000')) ----------------------------PhysicalOlapScan[household_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query74.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query74.out index 90fcd7d4edbc5a..6b6e625a96213e 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query74.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query74.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(13, 8)) / year_total), NULL) > if((year_total > 0.00), (cast(year_total as DECIMALV3(13, 8)) / year_total), NULL))) build RFs:RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id +----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(13, 8)) / t_w_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(13, 8)) / t_s_firstyear.year_total), NULL))) build RFs:RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF9 customer_id->c_customer_id;RF10 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id;RF11 customer_id->c_customer_id;RF15 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query75.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query75.out index e30d62d30ab824..0eda37accca422 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query75.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query75.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF6 ss_ticket_number->sr_ticket_number;RF7 ss_item_sk->sr_item_sk @@ -37,7 +37,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF10 ws_order_number->wr_order_number;RF11 ws_item_sk->wr_item_sk @@ -53,14 +53,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------------filter((item.i_category = 'Sports')) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter(d_year IN (2001, 2002)) +------------------------filter(date_dim.d_year IN (2001, 2002)) --------------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id +------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(curr_yr.sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(prev_yr.sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF12 i_brand_id->i_brand_id;RF13 i_class_id->i_class_id;RF14 i_category_id->i_category_id;RF15 i_manufact_id->i_manufact_id --------------filter((curr_yr.d_year = 2002)) ----------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 RF13 RF14 RF15 --------------filter((prev_yr.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query76.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query76.out index a6280f1f5d2cc2..b182393366a7ae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query76.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query76.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->ss_item_sk --------------------------PhysicalProject -----------------------------filter(ss_customer_sk IS NULL) +----------------------------filter(store_sales.ss_customer_sk IS NULL) ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF3 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -23,13 +23,13 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] apply RFs: RF1 ------------------------PhysicalProject ---------------------------filter(ws_promo_sk IS NULL) +--------------------------filter(web_sales.ws_promo_sk IS NULL) ----------------------------PhysicalOlapScan[web_sales] apply RFs: RF4 --------------------PhysicalDistribute[DistributionSpecExecutionAny] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk --------------------------PhysicalProject -----------------------------filter(cs_bill_customer_sk IS NULL) +----------------------------filter(catalog_sales.cs_bill_customer_sk IS NULL) ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF5 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query78.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query78.out index 0cb4bf14e4c5b8..aceea45035aad7 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query78.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query78.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)]) +----------filter(OR[(coalesce(ws.ws_qty, 0) > 0),(coalesce(cs.cs_qty, 0) > 0)]) ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=() --------------PhysicalProject ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=() diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query79.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query79.out index ae09791f4eb56a..419ef5d8072c00 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query79.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query79.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dow = 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dow = 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 7),(household_demographics.hd_vehicle_count > -1)]) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query8.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query8.out index e6b75f941a6c06..1e188ec784a6b4 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query8.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query8.out @@ -40,6 +40,6 @@ PhysicalResultSink ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +----------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ------------------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query81.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query81.out index dbccfac049d5b2..0dbad15942f272 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query81.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query81.out @@ -22,7 +22,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF5 ctr_state->ctr_state +------------hashJoin[INNER_JOIN broadcast] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF5 ctr_state->ctr_state --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->c_current_addr_sk ------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query82.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query82.out index e06e2ef7c59877..a6ab79b7c55bae 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query82.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query82.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 88.00) and (item.i_current_price >= 58.00) and i_manufact_id IN (259, 485, 559, 580)) +----------------------------filter((item.i_current_price <= 88.00) and (item.i_current_price >= 58.00) and item.i_manufact_id IN (259, 485, 559, 580)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2001-03-14') and (date_dim.d_date >= '2001-01-13')) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query83.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query83.out index e33fa4bc776525..e0ef2e6dbb9d56 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query83.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query83.out @@ -28,7 +28,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF8 ------------------------------------------PhysicalProject ---------------------------------------------filter(d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) +--------------------------------------------filter(date_dim.d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) ----------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[item] apply RFs: RF12 RF13 @@ -51,7 +51,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF4 ------------------------------------------PhysicalProject ---------------------------------------------filter(d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) +--------------------------------------------filter(date_dim.d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) ----------------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[item] apply RFs: RF14 @@ -74,7 +74,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) +----------------------------------------filter(date_dim.d_date IN ('2001-07-13', '2001-09-10', '2001-11-16')) ------------------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query85.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query85.out index dc606c9794bd3d..809f9e744278dc 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query85.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query85.out @@ -13,14 +13,14 @@ PhysicalResultSink --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF6 wr_returning_cdemo_sk->cd_demo_sk;RF7 cd_marital_status->cd_marital_status;RF8 cd_education_status->cd_education_status ------------------------PhysicalProject ---------------------------filter(cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U')) +--------------------------filter(cd2.cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd2.cd_marital_status IN ('D', 'S', 'U')) ----------------------------PhysicalOlapScan[customer_demographics(cd2)] apply RFs: RF6 RF7 RF8 ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF5 wp_web_page_sk->ws_web_page_sk ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->ca_address_sk +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('IA', 'NC', 'TX'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[customer_address.ca_state IN ('GA', 'WI', 'WV'),(web_sales.ws_net_profit >= 150.00)],AND[customer_address.ca_state IN ('KY', 'OK', 'VA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF4 wr_refunded_addr_sk->ca_address_sk --------------------------------PhysicalProject -----------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV')) +----------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('GA', 'IA', 'KY', 'NC', 'OK', 'TX', 'VA', 'WI', 'WV')) ------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4 --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary'),(web_sales.ws_sales_price >= 100.00),(web_sales.ws_sales_price <= 150.00)],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College'),(web_sales.ws_sales_price <= 100.00)],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree'),(web_sales.ws_sales_price >= 150.00)]]) build RFs:RF3 cd_demo_sk->wr_refunded_cdemo_sk @@ -37,7 +37,7 @@ PhysicalResultSink ----------------------------------------------filter((date_dim.d_year = 1998)) ------------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd_marital_status IN ('D', 'S', 'U')) +--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Primary')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'U'),(cd1.cd_education_status = 'Advanced Degree')]] and cd1.cd_education_status IN ('Advanced Degree', 'College', 'Primary') and cd1.cd_marital_status IN ('D', 'S', 'U')) ----------------------------------------PhysicalOlapScan[customer_demographics(cd1)] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_page] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query88.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query88.out index 3727b0265bea9d..acf802da3f4a3b 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query88.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query88.out @@ -23,7 +23,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -43,7 +43,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((store.s_store_name = 'ese')) @@ -63,7 +63,7 @@ PhysicalResultSink ----------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30)) ------------------------------------PhysicalOlapScan[time_dim] ----------------------------PhysicalProject -------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +------------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((store.s_store_name = 'ese')) @@ -83,7 +83,7 @@ PhysicalResultSink --------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30)) ----------------------------------PhysicalOlapScan[time_dim] --------------------------PhysicalProject -----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------------PhysicalOlapScan[household_demographics] ----------------------PhysicalProject ------------------------filter((store.s_store_name = 'ese')) @@ -103,7 +103,7 @@ PhysicalResultSink ------------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30)) --------------------------------PhysicalOlapScan[time_dim] ------------------------PhysicalProject ---------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((store.s_store_name = 'ese')) @@ -123,7 +123,7 @@ PhysicalResultSink ----------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30)) ------------------------------PhysicalOlapScan[time_dim] ----------------------PhysicalProject -------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +------------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) --------------------------PhysicalOlapScan[household_demographics] ------------------PhysicalProject --------------------filter((store.s_store_name = 'ese')) @@ -143,7 +143,7 @@ PhysicalResultSink --------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30)) ----------------------------PhysicalOlapScan[time_dim] --------------------PhysicalProject -----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +----------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ------------------------PhysicalOlapScan[household_demographics] ----------------PhysicalProject ------------------filter((store.s_store_name = 'ese')) @@ -163,7 +163,7 @@ PhysicalResultSink ------------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30)) --------------------------PhysicalOlapScan[time_dim] ------------------PhysicalProject ---------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and hd_dep_count IN (-1, 0, 3)) +--------------------filter((household_demographics.hd_vehicle_count <= 5) and OR[AND[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count <= 2)],AND[(household_demographics.hd_dep_count = -1),(household_demographics.hd_vehicle_count <= 1)],(household_demographics.hd_dep_count = 3)] and household_demographics.hd_dep_count IN (-1, 0, 3)) ----------------------PhysicalOlapScan[household_demographics] --------------PhysicalProject ----------------filter((store.s_store_name = 'ese')) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query89.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query89.out index af729e84567695..e2ab2c93469038 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query89.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query89.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (avg_monthly_sales = 0.0000)) and ((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) +------------filter(( not (tmp1.avg_monthly_sales = 0.0000)) and ((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------hashAgg[GLOBAL] @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('audio', 'history', 'school-uniforms')],AND[i_category IN ('Men', 'Shoes', 'Sports'),i_class IN ('pants', 'tennis', 'womens')]] and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('audio', 'history', 'school-uniforms')],AND[item.i_category IN ('Men', 'Shoes', 'Sports'),item.i_class IN ('pants', 'tennis', 'womens')]] and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Shoes', 'Sports') and item.i_class IN ('audio', 'history', 'pants', 'school-uniforms', 'tennis', 'womens')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query91.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query91.out index b6b6254efc444d..1ceeaab6bc0f46 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query91.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query91.out @@ -28,10 +28,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 RF1 ----------------------------------------PhysicalProject -------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +------------------------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) --------------------------------------------PhysicalOlapScan[customer_demographics] ------------------------------------PhysicalProject ---------------------------------------filter((hd_buy_potential like 'Unknown%')) +--------------------------------------filter((household_demographics.hd_buy_potential like 'Unknown%')) ----------------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((date_dim.d_moy = 12) and (date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query92.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query92.out index 0f325df9d725e5..cd7b8a423bbd93 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query92.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query92.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +------------filter((cast(web_sales.ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query94.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query94.out index 0c4c963c0b8fd0..e9553b57d751f3 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query94.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query94.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------hashAgg[GLOBAL] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF4 ws_order_number->ws_order_number +------------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF4 ws_order_number->ws_order_number --------------------PhysicalProject ----------------------PhysicalOlapScan[web_sales(ws2)] apply RFs: RF4 --------------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=() build RFs:RF3 ws_order_number->wr_order_number diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query95.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query95.out index ad211327e44889..f0f5bc65c652da 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query95.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF8 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query97.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query97.out index ce92c6d6b072d0..302d502ca82dcb 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query97.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query97.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->ss_sold_date_sk ----------------------------PhysicalProject -------------------------------filter(( not ss_sold_date_sk IS NULL)) +------------------------------filter(( not store_sales.ss_sold_date_sk IS NULL)) --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 ----------------------------PhysicalProject ------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199)) @@ -27,7 +27,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->cs_sold_date_sk ----------------------------PhysicalProject -------------------------------filter(( not cs_sold_date_sk IS NULL)) +------------------------------filter(( not catalog_sales.cs_sold_date_sk IS NULL)) --------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 ----------------------------PhysicalProject ------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199)) diff --git a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query98.out b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query98.out index 4fd2b8c9253d66..b526a834c318bc 100644 --- a/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query98.out +++ b/regression-test/data/shape_check/tpcds_sf1000_nopkfk/shape/query98.out @@ -21,6 +21,6 @@ PhysicalResultSink --------------------------------filter((date_dim.d_date <= '1999-03-07') and (date_dim.d_date >= '1999-02-05')) ----------------------------------PhysicalOlapScan[date_dim] --------------------------PhysicalProject -----------------------------filter(i_category IN ('Jewelry', 'Men', 'Sports')) +----------------------------filter(item.i_category IN ('Jewelry', 'Men', 'Sports')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query1.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query1.out index 14f65949035764..37cd33474167b7 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query1.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query1.out @@ -18,7 +18,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF3 ctr_store_sk->ctr_store_sk;RF4 ctr_store_sk->s_store_sk +------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF3 ctr_store_sk->ctr_store_sk;RF4 ctr_store_sk->s_store_sk --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->ctr_store_sk ------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query10.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query10.out index 8a62f8ac381034..46972350fb50b1 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query10.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query10.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer(c)] apply RFs: RF4 RF5 --------------------------------PhysicalProject -----------------------------------filter(ca_county IN ('Bonneville County', 'Boone County', 'Brown County', 'Fillmore County', 'McPherson County')) +----------------------------------filter(ca.ca_county IN ('Bonneville County', 'Boone County', 'Brown County', 'Fillmore County', 'McPherson County')) ------------------------------------PhysicalOlapScan[customer_address(ca)] ----------------------------PhysicalOlapScan[customer_demographics] ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query11.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query11.out index 38efdacdee15df..c14460551e6b25 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query11.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query11.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000) > if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), 0.000000))) build RFs:RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id +----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(38, 8)) / t_w_firstyear.year_total), 0.000000) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(38, 8)) / t_s_firstyear.year_total), 0.000000))) build RFs:RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF9 customer_id->c_customer_id;RF10 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id;RF11 customer_id->c_customer_id;RF15 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query12.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query12.out index 18bfc6f804813b..437a65082a72f6 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query12.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query12.out @@ -17,7 +17,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 ----------------------------PhysicalProject -------------------------------filter(i_category IN ('Books', 'Electronics', 'Women')) +------------------------------filter(item.i_category IN ('Books', 'Electronics', 'Women')) --------------------------------PhysicalOlapScan[item] ------------------------PhysicalProject --------------------------filter((date_dim.d_date <= '1998-02-05') and (date_dim.d_date >= '1998-01-06')) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query13.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query13.out index d469c99784ab03..ea89031acaf694 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query13.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query13.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->ss_sold_date_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('MI', 'OK', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[ca_state IN ('NC', 'OH', 'WA'),(store_sales.ss_net_profit >= 150.00)],AND[ca_state IN ('FL', 'GA', 'MT'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->ss_addr_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('MI', 'OK', 'TX'),(store_sales.ss_net_profit >= 100.00),(store_sales.ss_net_profit <= 200.00)],AND[customer_address.ca_state IN ('NC', 'OH', 'WA'),(store_sales.ss_net_profit >= 150.00)],AND[customer_address.ca_state IN ('FL', 'GA', 'MT'),(store_sales.ss_net_profit <= 250.00)]]) build RFs:RF3 ca_address_sk->ss_addr_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=(OR[AND[(household_demographics.hd_dep_count = 1),OR[AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00)]]],AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Secondary'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00),(household_demographics.hd_dep_count = 3)]]) build RFs:RF2 hd_demo_sk->ss_hdemo_sk --------------------PhysicalProject @@ -20,13 +20,13 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Secondary')]] and cd_education_status IN ('College', 'Primary', 'Secondary') and cd_marital_status IN ('D', 'U', 'W')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')],AND[(customer_demographics.cd_marital_status = 'U'),(customer_demographics.cd_education_status = 'Secondary')]] and customer_demographics.cd_education_status IN ('College', 'Primary', 'Secondary') and customer_demographics.cd_marital_status IN ('D', 'U', 'W')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter(hd_dep_count IN (1, 3)) +----------------------filter(household_demographics.hd_dep_count IN (1, 3)) ------------------------PhysicalOlapScan[household_demographics] ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('FL', 'GA', 'MI', 'MT', 'NC', 'OH', 'OK', 'TX', 'WA')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('FL', 'GA', 'MI', 'MT', 'NC', 'OH', 'OK', 'TX', 'WA')) --------------------PhysicalOlapScan[customer_address] ------------PhysicalProject --------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query15.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query15.out index 24929b034a06e5..b41dd36ee22736 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query15.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query15.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->cs_sold_date_sk ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF1 ca_address_sk->c_current_addr_sk +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),customer_address.ca_state IN ('CA', 'GA', 'WA'),(catalog_sales.cs_sales_price > 500.00)]) build RFs:RF1 ca_address_sk->c_current_addr_sk ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF0 c_customer_sk->cs_bill_customer_sk --------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query16.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query16.out index 9fb0d1a2f001e2..1317473b732323 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query16.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query16.out @@ -16,7 +16,7 @@ PhysicalResultSink --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cs1.cs_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->cs_ship_date_sk ----------------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((cs1.cs_order_number = cr1.cr_order_number)) otherCondition=() ------------------------------PhysicalProject ---------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs_warehouse_sk = cs_warehouse_sk))) build RFs:RF0 cs_order_number->cs_order_number +--------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((cs1.cs_order_number = cs2.cs_order_number)) otherCondition=(( not (cs1.cs_warehouse_sk = cs2.cs_warehouse_sk))) build RFs:RF0 cs_order_number->cs_order_number ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[catalog_sales(cs2)] apply RFs: RF0 ----------------------------------PhysicalProject @@ -30,6 +30,6 @@ PhysicalResultSink --------------------------filter((customer_address.ca_state = 'IL')) ----------------------------PhysicalOlapScan[customer_address] --------------------PhysicalProject -----------------------filter(cc_county IN ('Bronx County', 'Maverick County', 'Mesa County', 'Raleigh County', 'Richland County')) +----------------------filter(call_center.cc_county IN ('Bronx County', 'Maverick County', 'Mesa County', 'Raleigh County', 'Richland County')) ------------------------PhysicalOlapScan[call_center] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query17.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query17.out index ae118539f89968..797a5131ebf1a4 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query17.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query17.out @@ -34,10 +34,10 @@ PhysicalResultSink ----------------------------------filter((d1.d_quarter_name = '2000Q1')) ------------------------------------PhysicalOlapScan[date_dim(d1)] ----------------------------PhysicalProject -------------------------------filter(d_quarter_name IN ('2000Q1', '2000Q2', '2000Q3')) +------------------------------filter(d2.d_quarter_name IN ('2000Q1', '2000Q2', '2000Q3')) --------------------------------PhysicalOlapScan[date_dim(d2)] ------------------------PhysicalProject ---------------------------filter(d_quarter_name IN ('2000Q1', '2000Q2', '2000Q3')) +--------------------------filter(d3.d_quarter_name IN ('2000Q1', '2000Q2', '2000Q3')) ----------------------------PhysicalOlapScan[date_dim(d3)] --------------------PhysicalProject ----------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query18.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query18.out index c493cd2583d155..72baf9aa1864f9 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query18.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query18.out @@ -24,7 +24,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF2 RF5 RF6 ------------------------------------------PhysicalProject ---------------------------------------------filter(c_birth_month IN (1, 4, 5, 7, 8, 9)) +--------------------------------------------filter(customer.c_birth_month IN (1, 4, 5, 7, 8, 9)) ----------------------------------------------PhysicalOlapScan[customer] apply RFs: RF1 RF3 --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[customer_demographics(cd2)] @@ -32,7 +32,7 @@ PhysicalResultSink ------------------------------------filter((cd1.cd_education_status = 'Unknown') and (cd1.cd_gender = 'M')) --------------------------------------PhysicalOlapScan[customer_demographics(cd1)] ------------------------------PhysicalProject ---------------------------------filter(ca_state IN ('AL', 'AR', 'GA', 'MS', 'NC', 'TX', 'WV')) +--------------------------------filter(customer_address.ca_state IN ('AL', 'AR', 'GA', 'MS', 'NC', 'TX', 'WV')) ----------------------------------PhysicalOlapScan[customer_address] apply RFs: RF4 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query19.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query19.out index 911282ce776b98..41578fe154969b 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query19.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query19.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(ca_zip, 1, 5) = substring(s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=(( not (substring(customer_address.ca_zip, 1, 5) = substring(store.s_zip, 1, 5)))) build RFs:RF4 s_store_sk->ss_store_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ss_item_sk ------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query20.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query20.out index 1cfeb6f295e180..9963584666b207 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query20.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query20.out @@ -17,7 +17,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 ----------------------------PhysicalProject -------------------------------filter(i_category IN ('Children', 'Electronics', 'Shoes')) +------------------------------filter(item.i_category IN ('Children', 'Electronics', 'Shoes')) --------------------------------PhysicalOlapScan[item] ------------------------PhysicalProject --------------------------filter((date_dim.d_date <= '2001-04-13') and (date_dim.d_date >= '2001-03-14')) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query21.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query21.out index 86d80b63c1e777..e55f79a8bd48e2 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query21.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query21.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)) <= 1.5) and (if((inv_before > 0), (cast(inv_after as DOUBLE) / cast(inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) +--------filter(((cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)) <= 1.5) and (if((x.inv_before > 0), (cast(x.inv_after as DOUBLE) / cast(x.inv_before as DOUBLE)), NULL) >= cast((2.000000 / 3.0) as DOUBLE)) and (x.inv_before > 0)) ----------hashAgg[GLOBAL] ------------PhysicalDistribute[DistributionSpecHash] --------------hashAgg[LOCAL] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query23.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query23.out index c58a8c568fdeed..dec5ea4f48b600 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query23.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query23.out @@ -16,7 +16,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[item] ------------------PhysicalProject ---------------------filter(d_year IN (2000, 2001, 2002, 2003)) +--------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ----------------------PhysicalOlapScan[date_dim] --PhysicalCteAnchor ( cteId=CTEId#2 ) ----PhysicalCteProducer ( cteId=CTEId#2 ) @@ -45,7 +45,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[customer] --------------------------PhysicalProject -----------------------------filter(d_year IN (2000, 2001, 2002, 2003)) +----------------------------filter(date_dim.d_year IN (2000, 2001, 2002, 2003)) ------------------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalLimit[GLOBAL] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query24.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query24.out index 0bb8ded43874a8..ec6004a05d88c8 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query24.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query24.out @@ -11,7 +11,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->ss_item_sk;RF5 i_item_sk->sr_item_sk --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF3 ca_address_sk->c_current_addr_sk +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (customer.c_birth_country = upper(customer_address.ca_country)))) build RFs:RF3 ca_address_sk->c_current_addr_sk ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->ss_customer_sk ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query27.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query27.out index 2aae25d99f08dc..6dbfadf53085ea 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query27.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query27.out @@ -28,6 +28,6 @@ PhysicalResultSink ----------------------------filter((date_dim.d_year = 2000)) ------------------------------PhysicalOlapScan[date_dim] ----------------------PhysicalProject -------------------------filter(s_state IN ('AL', 'FL', 'IN', 'NY', 'OH', 'SC')) +------------------------filter(store.s_state IN ('AL', 'FL', 'IN', 'NY', 'OH', 'SC')) --------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query29.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query29.out index df724e87b6d116..eddc3461f42fa4 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query29.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query29.out @@ -36,7 +36,7 @@ PhysicalResultSink ----------------------------filter((d2.d_moy <= 7) and (d2.d_moy >= 4) and (d2.d_year = 1998)) ------------------------------PhysicalOlapScan[date_dim(d2)] ----------------------PhysicalProject -------------------------filter(d_year IN (1998, 1999, 2000)) +------------------------filter(d3.d_year IN (1998, 1999, 2000)) --------------------------PhysicalOlapScan[date_dim(d3)] ------------------PhysicalProject --------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query30.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query30.out index cd1594613a40db..475f71feae9940 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query30.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query30.out @@ -26,7 +26,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------PhysicalProject ----------------hashJoin[INNER_JOIN shuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->ctr_customer_sk ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_state->ctr_state +--------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF2 ctr_state->ctr_state ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF2 RF3 ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query31.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query31.out index fd17a6973fe4d6..0e6bccd4cc3388 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query31.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query31.out @@ -15,7 +15,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject -------------------filter((ss.d_year = 1999) and d_qoy IN (1, 2, 3)) +------------------filter((ss.d_year = 1999) and ss.d_qoy IN (1, 2, 3)) --------------------PhysicalOlapScan[date_dim] --PhysicalCteAnchor ( cteId=CTEId#1 ) ----PhysicalCteProducer ( cteId=CTEId#1 ) @@ -32,13 +32,13 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer_address] ------------------PhysicalProject ---------------------filter((ws.d_year = 1999) and d_qoy IN (1, 2, 3)) +--------------------filter((ws.d_year = 1999) and ws.d_qoy IN (1, 2, 3)) ----------------------PhysicalOlapScan[date_dim] ----PhysicalResultSink ------PhysicalProject ---------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF13 ca_county->ca_county;RF14 ca_county->ca_county;RF15 ca_county->ca_county;RF16 ca_county->ca_county +--------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws3.ca_county)) otherCondition=((if((ws2.web_sales > 0.00), (cast(ws3.web_sales as DECIMALV3(38, 8)) / ws2.web_sales), NULL) > if((ss2.store_sales > 0.00), (cast(ss3.store_sales as DECIMALV3(38, 8)) / ss2.store_sales), NULL))) build RFs:RF13 ca_county->ca_county;RF14 ca_county->ca_county;RF15 ca_county->ca_county;RF16 ca_county->ca_county ----------PhysicalProject -------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((web_sales > 0.00), (cast(web_sales as DECIMALV3(38, 8)) / web_sales), NULL) > if((store_sales > 0.00), (cast(store_sales as DECIMALV3(38, 8)) / store_sales), NULL))) build RFs:RF9 ca_county->ca_county;RF10 ca_county->ca_county;RF11 ca_county->ca_county +------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ws1.ca_county = ws2.ca_county)) otherCondition=((if((ws1.web_sales > 0.00), (cast(ws2.web_sales as DECIMALV3(38, 8)) / ws1.web_sales), NULL) > if((ss1.store_sales > 0.00), (cast(ss2.store_sales as DECIMALV3(38, 8)) / ss1.store_sales), NULL))) build RFs:RF9 ca_county->ca_county;RF10 ca_county->ca_county;RF11 ca_county->ca_county --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((ss1.ca_county = ws1.ca_county)) otherCondition=() build RFs:RF6 ca_county->ca_county;RF7 ca_county->ca_county;RF12 ca_county->ca_county;RF17 ca_county->ca_county ----------------PhysicalProject ------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ss2.ca_county = ss3.ca_county)) otherCondition=() build RFs:RF5 ca_county->ca_county diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query32.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query32.out index 6e5339ef4893be..d4d14c3b3c28bd 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query32.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query32.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------filter((cast(cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +--------------filter((cast(catalog_sales.cs_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(cs_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) ----------------PhysicalWindow ------------------PhysicalQuickSort[LOCAL_SORT] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query34.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query34.out index 09fee5adee28fd..609092e88d7ce4 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query34.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query34.out @@ -19,13 +19,13 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dom <= 28) and (date_dim.d_dom >= 1) and OR[(date_dim.d_dom <= 3),(date_dim.d_dom >= 25)] and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(s_county IN ('Arthur County', 'Halifax County', 'Lunenburg County', 'Oglethorpe County', 'Perry County', 'Salem County', 'Sumner County', 'Terrell County')) +------------------------------filter(store.s_county IN ('Arthur County', 'Halifax County', 'Lunenburg County', 'Oglethorpe County', 'Perry County', 'Salem County', 'Sumner County', 'Terrell County')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('>10000', 'Unknown')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.2) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('>10000', 'Unknown')) ----------------------------PhysicalOlapScan[household_demographics] ------------PhysicalProject --------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query36.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query36.out index 2e1772b57b2361..4d358fbc09f07f 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query36.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query36.out @@ -28,6 +28,6 @@ PhysicalResultSink --------------------------------------filter((d1.d_year = 1999)) ----------------------------------------PhysicalOlapScan[date_dim(d1)] --------------------------------PhysicalProject -----------------------------------filter(s_state IN ('AL', 'FL', 'IN', 'LA', 'MI', 'MN', 'NM', 'TN')) +----------------------------------filter(store.s_state IN ('AL', 'FL', 'IN', 'LA', 'MI', 'MN', 'NM', 'TN')) ------------------------------------PhysicalOlapScan[store] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query37.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query37.out index b9394e9bb2b082..9783593182b398 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query37.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query37.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 69.00) and (item.i_current_price >= 39.00) and i_manufact_id IN (728, 765, 886, 889)) +----------------------------filter((item.i_current_price <= 69.00) and (item.i_current_price >= 39.00) and item.i_manufact_id IN (728, 765, 886, 889)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2001-03-17') and (date_dim.d_date >= '2001-01-16')) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query39.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query39.out index 7f7329516cabfd..b9745c9214f564 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query39.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter(( not (mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) +------filter(( not (foo.mean = 0.0)) and ((foo.stdev / foo.mean) > 1.0)) --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] @@ -19,7 +19,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------PhysicalOlapScan[warehouse] ------------------PhysicalProject ---------------------filter((date_dim.d_year = 2000) and d_moy IN (2, 3)) +--------------------filter((date_dim.d_year = 2000) and inv.d_moy IN (2, 3)) ----------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query4.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query4.out index 543f6e2d4eb5d6..cdb41ec291ddc7 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query4.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query4.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id;RF27 customer_id->c_customer_id;RF28 customer_id->c_customer_id;RF29 customer_id->c_customer_id +----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_w_firstyear.year_total > 0.000000), (cast(t_w_secyear.year_total as DECIMALV3(38, 16)) / t_w_firstyear.year_total), NULL))) build RFs:RF25 customer_id->c_customer_id;RF26 customer_id->c_customer_id;RF27 customer_id->c_customer_id;RF28 customer_id->c_customer_id;RF29 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF20 customer_id->c_customer_id;RF21 customer_id->c_customer_id;RF22 customer_id->c_customer_id;RF23 customer_id->c_customer_id ----------------PhysicalProject -------------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL) > if((year_total > 0.000000), (cast(year_total as DECIMALV3(38, 16)) / year_total), NULL))) build RFs:RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id;RF18 customer_id->c_customer_id +------------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_c_secyear.customer_id)) otherCondition=((if((t_c_firstyear.year_total > 0.000000), (cast(t_c_secyear.year_total as DECIMALV3(38, 16)) / t_c_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.000000), (cast(t_s_secyear.year_total as DECIMALV3(38, 16)) / t_s_firstyear.year_total), NULL))) build RFs:RF16 customer_id->c_customer_id;RF17 customer_id->c_customer_id;RF18 customer_id->c_customer_id --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_c_firstyear.customer_id)) otherCondition=() build RFs:RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF12 customer_id->c_customer_id;RF15 customer_id->c_customer_id;RF19 customer_id->c_customer_id;RF24 customer_id->c_customer_id;RF30 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query41.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query41.out index 8456ee845afccf..9f24d6d0ca0935 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query41.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query41.out @@ -18,6 +18,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(OR[AND[(item.i_category = 'Men'),i_size IN ('N/A', 'large', 'medium', 'small'),OR[AND[i_size IN ('large', 'medium'),i_color IN ('cornflower', 'cyan', 'firebrick', 'maroon'),i_units IN ('Each', 'N/A', 'Oz', 'Pound'),OR[AND[i_color IN ('cornflower', 'firebrick'),i_units IN ('Oz', 'Pound')],AND[i_color IN ('cyan', 'maroon'),i_units IN ('Each', 'N/A')]]],AND[i_size IN ('N/A', 'small'),i_color IN ('lavender', 'magenta', 'papaya', 'slate'),i_units IN ('Bundle', 'Carton', 'Cup', 'Pallet'),OR[AND[i_color IN ('magenta', 'slate'),i_units IN ('Bundle', 'Carton')],AND[i_color IN ('lavender', 'papaya'),i_units IN ('Cup', 'Pallet')]]]]],AND[(item.i_category = 'Women'),i_size IN ('economy', 'large', 'medium', 'petite'),OR[AND[i_size IN ('large', 'medium'),i_color IN ('almond', 'frosted', 'rose', 'steel'),i_units IN ('Case', 'Gross', 'Lb', 'Tsp'),OR[AND[i_color IN ('frosted', 'rose'),i_units IN ('Gross', 'Lb')],AND[i_color IN ('almond', 'steel'),i_units IN ('Case', 'Tsp')]]],AND[i_size IN ('economy', 'petite'),i_color IN ('aquamarine', 'black', 'chocolate', 'purple'),i_units IN ('Box', 'Bunch', 'Dram', 'Gram'),OR[AND[i_color IN ('black', 'chocolate'),i_units IN ('Box', 'Dram')],AND[i_color IN ('aquamarine', 'purple'),i_units IN ('Bunch', 'Gram')]]]]]] and i_category IN ('Men', 'Women')) +------------------------------filter(OR[AND[(item.i_category = 'Men'),item.i_size IN ('N/A', 'large', 'medium', 'small'),OR[AND[item.i_size IN ('large', 'medium'),item.i_color IN ('cornflower', 'cyan', 'firebrick', 'maroon'),item.i_units IN ('Each', 'N/A', 'Oz', 'Pound'),OR[AND[item.i_color IN ('cornflower', 'firebrick'),item.i_units IN ('Oz', 'Pound')],AND[item.i_color IN ('cyan', 'maroon'),item.i_units IN ('Each', 'N/A')]]],AND[item.i_size IN ('N/A', 'small'),item.i_color IN ('lavender', 'magenta', 'papaya', 'slate'),item.i_units IN ('Bundle', 'Carton', 'Cup', 'Pallet'),OR[AND[item.i_color IN ('magenta', 'slate'),item.i_units IN ('Bundle', 'Carton')],AND[item.i_color IN ('lavender', 'papaya'),item.i_units IN ('Cup', 'Pallet')]]]]],AND[(item.i_category = 'Women'),item.i_size IN ('economy', 'large', 'medium', 'petite'),OR[AND[item.i_size IN ('large', 'medium'),item.i_color IN ('almond', 'frosted', 'rose', 'steel'),item.i_units IN ('Case', 'Gross', 'Lb', 'Tsp'),OR[AND[item.i_color IN ('frosted', 'rose'),item.i_units IN ('Gross', 'Lb')],AND[item.i_color IN ('almond', 'steel'),item.i_units IN ('Case', 'Tsp')]]],AND[item.i_size IN ('economy', 'petite'),item.i_color IN ('aquamarine', 'black', 'chocolate', 'purple'),item.i_units IN ('Box', 'Bunch', 'Dram', 'Gram'),OR[AND[item.i_color IN ('black', 'chocolate'),item.i_units IN ('Box', 'Dram')],AND[item.i_color IN ('aquamarine', 'purple'),item.i_units IN ('Bunch', 'Gram')]]]]]] and item.i_category IN ('Men', 'Women')) --------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query44.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query44.out index 20079a8f52b53a..aaac9669ab78a4 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query44.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query44.out @@ -38,7 +38,7 @@ PhysicalResultSink ----------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------hashAgg[LOCAL] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((store_sales.ss_store_sk = 366) and ss_cdemo_sk IS NULL) +----------------------------------------------------------filter((store_sales.ss_store_sk = 366) and store_sales.ss_cdemo_sk IS NULL) ------------------------------------------------------------PhysicalOlapScan[store_sales] ------------------------PhysicalProject --------------------------filter((V21.rnk < 11)) @@ -64,7 +64,7 @@ PhysicalResultSink ----------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------hashAgg[LOCAL] --------------------------------------------------------PhysicalProject -----------------------------------------------------------filter((store_sales.ss_store_sk = 366) and ss_cdemo_sk IS NULL) +----------------------------------------------------------filter((store_sales.ss_store_sk = 366) and store_sales.ss_cdemo_sk IS NULL) ------------------------------------------------------------PhysicalOlapScan[store_sales] ----------------PhysicalProject ------------------PhysicalLazyMaterializeOlapScan[item lazySlots:(i2.i_product_name)] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query45.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query45.out index 390771de3ab976..277de19dfd15f0 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query45.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query45.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------filter(OR[substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) +----------------filter(OR[substring(customer_address.ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274'),$c$1]) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ws_item_sk --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->ws_sold_date_sk @@ -30,6 +30,6 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[item] ------------------------PhysicalProject ---------------------------filter(i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) +--------------------------filter(item.i_item_sk IN (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)) ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query46.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query46.out index 0b6c01cb8579bf..dcc0a6f0bbe764 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query46.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query46.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk +----------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk ------------PhysicalProject --------------hashJoin[INNER_JOIN shuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->ss_customer_sk ----------------PhysicalProject @@ -21,10 +21,10 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 RF4 ------------------------------------PhysicalProject ---------------------------------------filter(d_dow IN (0, 6) and d_year IN (2000, 2001, 2002)) +--------------------------------------filter(date_dim.d_dow IN (0, 6) and date_dim.d_year IN (2000, 2001, 2002)) ----------------------------------------PhysicalOlapScan[date_dim] --------------------------------PhysicalProject -----------------------------------filter(s_city IN ('Fairview', 'Farmington', 'Five Forks', 'Oakland', 'Winchester')) +----------------------------------filter(store.s_city IN ('Fairview', 'Farmington', 'Five Forks', 'Oakland', 'Winchester')) ------------------------------------PhysicalOlapScan[store] ----------------------------PhysicalProject ------------------------------filter(OR[(household_demographics.hd_dep_count = 0),(household_demographics.hd_vehicle_count = 1)]) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query47.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query47.out index 13e9ad62153f99..16f72062f9662f 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query47.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query47.out @@ -20,7 +20,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[item] --------------------------PhysicalProject -----------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000)) +----------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------PhysicalOlapScan[date_dim] ----------------------PhysicalProject ------------------------PhysicalOlapScan[store] @@ -37,6 +37,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1)) and (v1.s_company_name = v1_lag.s_company_name) and (v1.s_store_name = v1_lag.s_store_name)) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 s_store_name->s_store_name;RF6 s_company_name->s_company_name;RF7 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 RF7 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query48.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query48.out index 4b84a40840c182..2ee15514de1ec5 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query48.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query48.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->ss_sold_date_sk ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[ca_state IN ('GA', 'MI', 'NH'),(store_sales.ss_net_profit <= 2000.00)],AND[ca_state IN ('KY', 'SD', 'TX'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[ca_state IN ('FL', 'NY', 'OH'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->ss_addr_sk +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('GA', 'MI', 'NH'),(store_sales.ss_net_profit <= 2000.00)],AND[customer_address.ca_state IN ('KY', 'SD', 'TX'),(store_sales.ss_net_profit >= 150.00),(store_sales.ss_net_profit <= 3000.00)],AND[customer_address.ca_state IN ('FL', 'NY', 'OH'),(store_sales.ss_net_profit >= 50.00)]]) build RFs:RF2 ca_address_sk->ss_addr_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown'),(store_sales.ss_sales_price >= 100.00),(store_sales.ss_sales_price <= 150.00)],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College'),(store_sales.ss_sales_price <= 100.00)],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary'),(store_sales.ss_sales_price >= 150.00)]]) build RFs:RF1 cd_demo_sk->ss_cdemo_sk --------------------PhysicalProject @@ -18,10 +18,10 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[store] --------------------PhysicalProject -----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')]] and cd_education_status IN ('College', 'Primary', 'Unknown') and cd_marital_status IN ('D', 'M', 'W')) +----------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'College')],AND[(customer_demographics.cd_marital_status = 'D'),(customer_demographics.cd_education_status = 'Primary')]] and customer_demographics.cd_education_status IN ('College', 'Primary', 'Unknown') and customer_demographics.cd_marital_status IN ('D', 'M', 'W')) ------------------------PhysicalOlapScan[customer_demographics] ----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('FL', 'GA', 'KY', 'MI', 'NH', 'NY', 'OH', 'SD', 'TX')) +------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('FL', 'GA', 'KY', 'MI', 'NH', 'NY', 'OH', 'SD', 'TX')) --------------------PhysicalOlapScan[customer_address] ------------PhysicalProject --------------filter((date_dim.d_year = 1998)) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query53.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query53.out index 7210f0d15afc33..a47274c6f9b978 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query53.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query53.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_quarterly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_quarterly_sales) > 0.100000) and (tmp1.avg_quarterly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalProject @@ -20,11 +20,11 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------PhysicalProject -------------------------------------filter(d_month_seq IN (1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223)) +------------------------------------filter(date_dim.d_month_seq IN (1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223)) --------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[store] --------------------------PhysicalProject -----------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +----------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query56.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query56.out index 6947860300ab20..d7734639c25827 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query56.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query56.out @@ -27,7 +27,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('chiffon', 'lace', 'smoke')) +------------------------------------filter(item.i_color IN ('chiffon', 'lace', 'smoke')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2001)) @@ -51,7 +51,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF4 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('chiffon', 'lace', 'smoke')) +------------------------------------filter(item.i_color IN ('chiffon', 'lace', 'smoke')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2001)) @@ -75,7 +75,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------------------------PhysicalProject -------------------------------------filter(i_color IN ('chiffon', 'lace', 'smoke')) +------------------------------------filter(item.i_color IN ('chiffon', 'lace', 'smoke')) --------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query57.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query57.out index 76df008f13f025..41ca0aa47d7264 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query57.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query57.out @@ -20,7 +20,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[item] --------------------------PhysicalProject -----------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and d_year IN (1998, 1999, 2000)) +----------------------------filter(OR[(date_dim.d_year = 1999),AND[(date_dim.d_year = 1998),(date_dim.d_moy = 12)],AND[(date_dim.d_year = 2000),(date_dim.d_moy = 1)]] and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------PhysicalOlapScan[date_dim] ----------------------PhysicalProject ------------------------PhysicalOlapScan[call_center] @@ -37,6 +37,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((v1.cc_name = v1_lag.cc_name) and (v1.i_brand = v1_lag.i_brand) and (v1.i_category = v1_lag.i_category) and (v1.rn = expr_(rn + 1))) otherCondition=() build RFs:RF3 i_category->i_category;RF4 i_brand->i_brand;RF5 cc_name->cc_name;RF6 rn->(rn + 1) --------------------PhysicalProject ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF3 RF4 RF5 RF6 ---------------------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) +--------------------filter(((cast(abs((v2.sum_sales - cast(v2.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / v2.avg_monthly_sales) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query58.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query58.out index 6b1bb701623ac3..2361e22bc59ce9 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query58.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query58.out @@ -6,9 +6,9 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id;RF14 item_id->i_item_id +------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = ws_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * ws_items.ws_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * ws_items.ws_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev)) and (cast(ws_items.ws_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev))) build RFs:RF13 item_id->i_item_id;RF14 item_id->i_item_id --------------PhysicalProject -----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF12 item_id->i_item_id +----------------hashJoin[INNER_JOIN colocated] hashCondition=((ss_items.item_id = cs_items.item_id)) otherCondition=((cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) <= (1.1 * ss_items.ss_item_rev)) and (cast(cs_items.cs_item_rev as DECIMALV3(38, 3)) >= (0.9 * ss_items.ss_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) <= (1.1 * cs_items.cs_item_rev)) and (cast(ss_items.ss_item_rev as DECIMALV3(38, 3)) >= (0.9 * cs_items.cs_item_rev))) build RFs:RF12 item_id->i_item_id ------------------PhysicalProject --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query6.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query6.out index 56bc77a1e31835..f7e05113ffbebb 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query6.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query6.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF5 i_category->i_category +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((j.i_category = i.i_category)) otherCondition=((cast(i.i_current_price as DECIMALV3(38, 5)) > (1.2 * avg(j.i_current_price)))) build RFs:RF5 i_category->i_category ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d.d_month_seq = date_dim.d_month_seq)) otherCondition=() build RFs:RF4 d_month_seq->d_month_seq --------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query63.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query63.out index 128f8b2bed6375..a259e65e6f5274 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query63.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query63.out @@ -4,7 +4,7 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter(((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) +--------filter(((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000) and (tmp1.avg_monthly_sales > 0.0000)) ----------PhysicalWindow ------------PhysicalQuickSort[LOCAL_SORT] --------------PhysicalProject @@ -20,11 +20,11 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------------------PhysicalProject -------------------------------------filter(d_month_seq IN (1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222)) +------------------------------------filter(date_dim.d_month_seq IN (1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222)) --------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[store] --------------------------PhysicalProject -----------------------------filter(OR[AND[i_category IN ('Books', 'Children', 'Electronics'),i_class IN ('personal', 'portable', 'reference', 'self-help'),i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[i_category IN ('Men', 'Music', 'Women'),i_class IN ('accessories', 'classical', 'fragrances', 'pants'),i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) +----------------------------filter(OR[AND[item.i_category IN ('Books', 'Children', 'Electronics'),item.i_class IN ('personal', 'portable', 'reference', 'self-help'),item.i_brand IN ('exportiunivamalg #9', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9')],AND[item.i_category IN ('Men', 'Music', 'Women'),item.i_class IN ('accessories', 'classical', 'fragrances', 'pants'),item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'importoamalg #1')]] and item.i_brand IN ('amalgimporto #1', 'edu packscholar #1', 'exportiimporto #1', 'exportiunivamalg #9', 'importoamalg #1', 'scholaramalgamalg #14', 'scholaramalgamalg #7', 'scholaramalgamalg #9') and item.i_category IN ('Books', 'Children', 'Electronics', 'Men', 'Music', 'Women') and item.i_class IN ('accessories', 'classical', 'fragrances', 'pants', 'personal', 'portable', 'reference', 'self-help')) ------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query64.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query64.out index 8391d3c582192d..07814e249d591e 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query64.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query64.out @@ -23,7 +23,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF13 p_promo_sk->ss_promo_sk --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF12 cd_demo_sk->c_current_cdemo_sk +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd1.cd_marital_status = cd2.cd_marital_status))) build RFs:RF12 cd_demo_sk->c_current_cdemo_sk ------------------------------------------------PhysicalProject --------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() build RFs:RF11 cd_demo_sk->ss_cdemo_sk ----------------------------------------------------PhysicalProject @@ -56,7 +56,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ------------------------------------------------------------------------------------------PhysicalProject --------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF23 ------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------filter(d_year IN (1999, 2000)) +--------------------------------------------------------------------------filter(d1.d_year IN (1999, 2000)) ----------------------------------------------------------------------------PhysicalOlapScan[date_dim(d1)] --------------------------------------------------------------------PhysicalProject ----------------------------------------------------------------------PhysicalOlapScan[store] @@ -85,7 +85,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------PhysicalProject ----------------------PhysicalOlapScan[income_band(ib2)] ----------------PhysicalProject -------------------filter((item.i_current_price <= 90.00) and (item.i_current_price >= 81.00) and i_color IN ('azure', 'blush', 'gainsboro', 'hot', 'lemon', 'misty')) +------------------filter((item.i_current_price <= 90.00) and (item.i_current_price >= 81.00) and item.i_color IN ('azure', 'blush', 'gainsboro', 'hot', 'lemon', 'misty')) --------------------PhysicalOlapScan[item] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query65.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query65.out index 0ab5e1ce17e2e5..a719f1ad597d69 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query65.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query65.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN colocated] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk +--------------hashJoin[INNER_JOIN colocated] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(sc.revenue as DECIMALV3(38, 5)) <= (0.1 * sb.ave))) build RFs:RF4 ss_store_sk->ss_store_sk;RF5 ss_store_sk->s_store_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((item.i_item_sk = sc.ss_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->ss_item_sk --------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query66.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query66.out index a387d72c582784..5386a35ae242ac 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query66.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query66.out @@ -32,7 +32,7 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_time <= 38253) and (time_dim.t_time >= 9453)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter(sm_carrier IN ('GERMA', 'MSC')) +--------------------------------filter(ship_mode.sm_carrier IN ('GERMA', 'MSC')) ----------------------------------PhysicalOlapScan[ship_mode] --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] @@ -56,6 +56,6 @@ PhysicalResultSink ------------------------------------filter((time_dim.t_time <= 38253) and (time_dim.t_time >= 9453)) --------------------------------------PhysicalOlapScan[time_dim] ------------------------------PhysicalProject ---------------------------------filter(sm_carrier IN ('GERMA', 'MSC')) +--------------------------------filter(ship_mode.sm_carrier IN ('GERMA', 'MSC')) ----------------------------------PhysicalOlapScan[ship_mode] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query68.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query68.out index 8386fadb8c3d46..45cbef9db23014 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query68.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query68.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalDistribute[DistributionSpecGather] ----------PhysicalTopN[LOCAL_SORT] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (current_addr.ca_city = dn.bought_city))) build RFs:RF5 ca_address_sk->c_current_addr_sk ----------------PhysicalProject ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->ss_customer_sk --------------------PhysicalProject @@ -23,10 +23,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 RF4 ----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1999, 2000, 2001)) +------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (1999, 2000, 2001)) --------------------------------------------PhysicalOlapScan[date_dim] ------------------------------------PhysicalProject ---------------------------------------filter(s_city IN ('Bethel', 'Pleasant Hill')) +--------------------------------------filter(store.s_city IN ('Bethel', 'Pleasant Hill')) ----------------------------------------PhysicalOlapScan[store] --------------------------------PhysicalProject ----------------------------------filter(OR[(household_demographics.hd_dep_count = 4),(household_demographics.hd_vehicle_count = 0)]) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query69.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query69.out index 42ee611959e902..1acf953c1d6955 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query69.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query69.out @@ -33,7 +33,7 @@ PhysicalResultSink ------------------------------------filter((date_dim.d_moy <= 4) and (date_dim.d_moy >= 2) and (date_dim.d_year = 2003)) --------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(ca_state IN ('AZ', 'MN', 'MO')) +------------------------------filter(ca.ca_state IN ('AZ', 'MN', 'MO')) --------------------------------PhysicalOlapScan[customer_address(ca)] ------------------------PhysicalProject --------------------------PhysicalOlapScan[customer_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query71.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query71.out index c92a00e55e2cda..6d78685fc37e39 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query71.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query71.out @@ -31,6 +31,6 @@ PhysicalResultSink --------------------------filter((item.i_manager_id = 1)) ----------------------------PhysicalOlapScan[item] --------------------PhysicalProject -----------------------filter(t_meal_time IN ('breakfast', 'dinner')) +----------------------filter(time_dim.t_meal_time IN ('breakfast', 'dinner')) ------------------------PhysicalOlapScan[time_dim] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query72.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query72.out index 2ee09e5822b908..6153b6c31d246f 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query72.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query72.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk) and (d1.d_week_seq = d2.d_week_seq)) otherCondition=((cast(d_date as DATETIMEV2(0)) > cast((cast(d_date as BIGINT) + 5) as DATETIMEV2(0)))) build RFs:RF8 d_week_seq->d_week_seq;RF9 d_date_sk->cs_sold_date_sk +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk) and (d1.d_week_seq = d2.d_week_seq)) otherCondition=((cast(d3.d_date as DATETIMEV2(0)) > cast((cast(d1.d_date as BIGINT) + 5) as DATETIMEV2(0)))) build RFs:RF8 d_week_seq->d_week_seq;RF9 d_date_sk->cs_sold_date_sk ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF7 hd_demo_sk->cs_bill_hdemo_sk ----------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query73.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query73.out index 003026b9b73874..87113780fe1902 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query73.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query73.out @@ -19,13 +19,13 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (2000, 2001, 2002)) +----------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and date_dim.d_year IN (2000, 2001, 2002)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject -------------------------------filter(s_county IN ('Bronx County', 'Furnas County', 'Lea County', 'Pennington County')) +------------------------------filter(store.s_county IN ('Bronx County', 'Furnas County', 'Lea County', 'Pennington County')) --------------------------------PhysicalOlapScan[store] ------------------------PhysicalProject ---------------------------filter(((cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and hd_buy_potential IN ('5001-10000', '>10000')) +--------------------------filter(((cast(household_demographics.hd_dep_count as DOUBLE) / cast(household_demographics.hd_vehicle_count as DOUBLE)) > 1.0) and (household_demographics.hd_vehicle_count > 0) and household_demographics.hd_buy_potential IN ('5001-10000', '>10000')) ----------------------------PhysicalOlapScan[household_demographics] ------------PhysicalProject --------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query74.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query74.out index 8f8d12e1bcd148..a0da552c788b50 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query74.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query74.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), NULL) > if((year_total > 0.00), (cast(year_total as DECIMALV3(38, 8)) / year_total), NULL))) build RFs:RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id +----------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_secyear.customer_id)) otherCondition=((if((t_w_firstyear.year_total > 0.00), (cast(t_w_secyear.year_total as DECIMALV3(38, 8)) / t_w_firstyear.year_total), NULL) > if((t_s_firstyear.year_total > 0.00), (cast(t_s_secyear.year_total as DECIMALV3(38, 8)) / t_s_firstyear.year_total), NULL))) build RFs:RF12 customer_id->c_customer_id;RF13 customer_id->c_customer_id;RF14 customer_id->c_customer_id ------------PhysicalProject --------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_firstyear.customer_id = t_w_firstyear.customer_id)) otherCondition=() build RFs:RF9 customer_id->c_customer_id;RF10 customer_id->c_customer_id ----------------hashJoin[INNER_JOIN colocated] hashCondition=((t_s_secyear.customer_id = t_s_firstyear.customer_id)) otherCondition=() build RFs:RF8 customer_id->c_customer_id;RF11 customer_id->c_customer_id;RF15 customer_id->c_customer_id diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query75.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query75.out index b9eb34e981a354..59c1190d9ba0f4 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query75.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query75.out @@ -21,7 +21,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------filter((item.i_category = 'Sports')) --------------------------PhysicalOlapScan[item] ------------------PhysicalProject ---------------------filter(d_year IN (2000, 2001)) +--------------------filter(date_dim.d_year IN (2000, 2001)) ----------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->ss_sold_date_sk @@ -37,7 +37,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------filter((item.i_category = 'Sports')) --------------------------PhysicalOlapScan[item] ------------------PhysicalProject ---------------------filter(d_year IN (2000, 2001)) +--------------------filter(date_dim.d_year IN (2000, 2001)) ----------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = web_sales.ws_sold_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->ws_sold_date_sk @@ -53,14 +53,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------filter((item.i_category = 'Sports')) --------------------------PhysicalOlapScan[item] ------------------PhysicalProject ---------------------filter(d_year IN (2000, 2001)) +--------------------filter(date_dim.d_year IN (2000, 2001)) ----------------------PhysicalOlapScan[date_dim] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF6 i_brand_id->i_brand_id;RF7 i_class_id->i_class_id;RF8 i_category_id->i_category_id;RF9 i_manufact_id->i_manufact_id +------------hashJoin[INNER_JOIN shuffle] hashCondition=((curr_yr.i_brand_id = prev_yr.i_brand_id) and (curr_yr.i_category_id = prev_yr.i_category_id) and (curr_yr.i_class_id = prev_yr.i_class_id) and (curr_yr.i_manufact_id = prev_yr.i_manufact_id)) otherCondition=(((cast(cast(curr_yr.sales_cnt as DECIMALV3(17, 2)) as DECIMALV3(23, 8)) / cast(prev_yr.sales_cnt as DECIMALV3(17, 2))) < 0.900000)) build RFs:RF6 i_brand_id->i_brand_id;RF7 i_class_id->i_class_id;RF8 i_category_id->i_category_id;RF9 i_manufact_id->i_manufact_id --------------filter((curr_yr.d_year = 2001)) ----------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF6 RF7 RF8 RF9 --------------filter((prev_yr.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query76.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query76.out index 68905673d6ef42..a5fc7395ce6e51 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query76.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query76.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->ss_item_sk --------------------------PhysicalProject -----------------------------filter(ss_customer_sk IS NULL) +----------------------------filter(store_sales.ss_customer_sk IS NULL) ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF3 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -22,7 +22,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->ws_item_sk --------------------------PhysicalProject -----------------------------filter(ws_ship_addr_sk IS NULL) +----------------------------filter(web_sales.ws_ship_addr_sk IS NULL) ------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF4 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] @@ -30,7 +30,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->cs_item_sk --------------------------PhysicalProject -----------------------------filter(cs_ship_mode_sk IS NULL) +----------------------------filter(catalog_sales.cs_ship_mode_sk IS NULL) ------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF5 --------------------------PhysicalProject ----------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query78.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query78.out index ded54520a62b9b..52d515e653c273 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query78.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query78.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------filter(OR[(coalesce(ws_qty, 0) > 0),(coalesce(cs_qty, 0) > 0)]) +----------filter(OR[(coalesce(ws.ws_qty, 0) > 0),(coalesce(cs.cs_qty, 0) > 0)]) ------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((cs.cs_customer_sk = ss.ss_customer_sk) and (cs.cs_item_sk = ss.ss_item_sk) and (cs.cs_sold_year = ss.ss_sold_year)) otherCondition=() --------------PhysicalProject ----------------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((ws.ws_customer_sk = ss.ss_customer_sk) and (ws.ws_item_sk = ss.ss_item_sk) and (ws.ws_sold_year = ss.ss_sold_year)) otherCondition=() diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query79.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query79.out index 9621b2c76c8473..f75010f46cbc93 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query79.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query79.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 --------------------------------PhysicalProject -----------------------------------filter((date_dim.d_dow = 1) and d_year IN (1998, 1999, 2000)) +----------------------------------filter((date_dim.d_dow = 1) and date_dim.d_year IN (1998, 1999, 2000)) ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------filter((store.s_number_employees <= 295) and (store.s_number_employees >= 200)) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query8.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query8.out index bcade99b07dee3..776c25810c861c 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query8.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query8.out @@ -34,10 +34,10 @@ PhysicalResultSink ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 --------------------------------------PhysicalProject -----------------------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +----------------------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ------------------------------------------PhysicalOlapScan[customer_address] ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------PhysicalProject ---------------------------filter(substring(ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) +--------------------------filter(substring(customer_address.ca_zip, 1, 5) IN ('10298', '10374', '10425', '11340', '11489', '11618', '11652', '11686', '11855', '11912', '12197', '12318', '12320', '12350', '13086', '13123', '13261', '13338', '13376', '13378', '13443', '13844', '13869', '13918', '14073', '14155', '14196', '14242', '14312', '14440', '14530', '14851', '15371', '15475', '15543', '15734', '15751', '15782', '15794', '16005', '16226', '16364', '16515', '16704', '16791', '16891', '17167', '17193', '17291', '17672', '17819', '17879', '17895', '18218', '18360', '18367', '18410', '18421', '18434', '18569', '18700', '18767', '18829', '18884', '19326', '19444', '19489', '19753', '19833', '19988', '20244', '20317', '20534', '20601', '20712', '21060', '21094', '21204', '21231', '21343', '21727', '21800', '21814', '22728', '22815', '22911', '23065', '23952', '24227', '24255', '24286', '24594', '24660', '24891', '24987', '25115', '25178', '25214', '25264', '25333', '25494', '25717', '25973', '26217', '26689', '27052', '27116', '27156', '27287', '27369', '27385', '27413', '27642', '27700', '28055', '28239', '28571', '28577', '28810', '29086', '29392', '29450', '29752', '29818', '30106', '30415', '30621', '31013', '31016', '31655', '31830', '32489', '32669', '32754', '32919', '32958', '32961', '33113', '33122', '33159', '33467', '33562', '33773', '33869', '34306', '34473', '34594', '34948', '34972', '35076', '35390', '35834', '35863', '35926', '36201', '36335', '36430', '36479', '37119', '37788', '37914', '38353', '38607', '38919', '39214', '39459', '39500', '39503', '40146', '40936', '40979', '41162', '41232', '41255', '41331', '41351', '41352', '41419', '41807', '41836', '41967', '42361', '43432', '43639', '43830', '43933', '44529', '45266', '45484', '45533', '45645', '45676', '45859', '46081', '46131', '46507', '47289', '47369', '47529', '47602', '47770', '48017', '48162', '48333', '48530', '48567', '49101', '49130', '49140', '49211', '49230', '49254', '49472', '50412', '50632', '50636', '50679', '50788', '51089', '51184', '51195', '51634', '51717', '51766', '51782', '51793', '51933', '52094', '52301', '52389', '52868', '53163', '53535', '53565', '54010', '54207', '54364', '54558', '54585', '55233', '55349', '56224', '56355', '56436', '56455', '56600', '56877', '57025', '57553', '57631', '57649', '57839', '58032', '58058', '58062', '58117', '58218', '58412', '58454', '58581', '59004', '59080', '59130', '59226', '59345', '59386', '59494', '59852', '60083', '60298', '60560', '60624', '60736', '61527', '61794', '61860', '61997', '62361', '62585', '62878', '63073', '63180', '63193', '63294', '63792', '63991', '64592', '65148', '65177', '65501', '66057', '66943', '67881', '67975', '67998', '68101', '68293', '68341', '68605', '68730', '68770', '68843', '68852', '68908', '69280', '69952', '69998', '70041', '70070', '70073', '70450', '71144', '71256', '71286', '71836', '71948', '71954', '71997', '72592', '72991', '73021', '73108', '73134', '73146', '73219', '73873', '74686', '75660', '75675', '75742', '75752', '77454', '77817', '78093', '78366', '79077', '79658', '80332', '80846', '81003', '81070', '81084', '81335', '81504', '81755', '81963', '82080', '82602', '82620', '83041', '83086', '83583', '83647', '83833', '83910', '83986', '84247', '84680', '84844', '84919', '85066', '85761', '86057', '86379', '86709', '88086', '88137', '88217', '89193', '89338', '90209', '90229', '90669', '91110', '91894', '92292', '92380', '92645', '92696', '93498', '94791', '94835', '94898', '95042', '95430', '95464', '95694', '96435', '96560', '97173', '97462', '98069', '98072', '98338', '98533', '98569', '98584', '98862', '99060', '99132')) ----------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query81.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query81.out index 9dad801ff1e29c..7464f4e732fb9d 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query81.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query81.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalProject ------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF4 ca_address_sk->c_current_addr_sk --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF3 ctr_state->ctr_state +----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr1.ctr_total_return as DECIMALV3(38, 5)) > (avg(ctr_total_return) * 1.2))) build RFs:RF3 ctr_state->ctr_state ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->ctr_customer_sk ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF2 RF3 diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query82.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query82.out index c46e2b1ec2b5c9..cd01ee62dfdd3b 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query82.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query82.out @@ -19,7 +19,7 @@ PhysicalResultSink ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((item.i_current_price <= 79.00) and (item.i_current_price >= 49.00) and i_manufact_id IN (17, 292, 675, 80)) +----------------------------filter((item.i_current_price <= 79.00) and (item.i_current_price >= 49.00) and item.i_manufact_id IN (17, 292, 675, 80)) ------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject ------------------------filter((date_dim.d_date <= '2001-03-29') and (date_dim.d_date >= '2001-01-28')) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query83.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query83.out index 4cdbf165feaa50..d831763dcd545a 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query83.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query83.out @@ -30,7 +30,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF8 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2000-06-17', '2000-08-22', '2000-11-17')) +----------------------------------------filter(date_dim.d_date IN ('2000-06-17', '2000-08-22', '2000-11-17')) ------------------------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject --------------------hashAgg[GLOBAL] @@ -53,7 +53,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF4 --------------------------------------PhysicalProject -----------------------------------------filter(d_date IN ('2000-06-17', '2000-08-22', '2000-11-17')) +----------------------------------------filter(date_dim.d_date IN ('2000-06-17', '2000-08-22', '2000-11-17')) ------------------------------------------PhysicalOlapScan[date_dim] --------------PhysicalProject ----------------hashAgg[GLOBAL] @@ -76,6 +76,6 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF0 ----------------------------------PhysicalProject -------------------------------------filter(d_date IN ('2000-06-17', '2000-08-22', '2000-11-17')) +------------------------------------filter(date_dim.d_date IN ('2000-06-17', '2000-08-22', '2000-11-17')) --------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query85.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query85.out index b9e09722a864b7..241ce61fa7b68f 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query85.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query85.out @@ -13,7 +13,7 @@ PhysicalResultSink --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->ws_sold_date_sk ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[ca_state IN ('CA', 'TX', 'VA'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[ca_state IN ('AR', 'MO', 'NE'),(web_sales.ws_net_profit >= 150.00)],AND[ca_state IN ('IA', 'MS', 'WA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->wr_refunded_addr_sk +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=(OR[AND[customer_address.ca_state IN ('CA', 'TX', 'VA'),(web_sales.ws_net_profit >= 100.00),(web_sales.ws_net_profit <= 200.00)],AND[customer_address.ca_state IN ('AR', 'MO', 'NE'),(web_sales.ws_net_profit >= 150.00)],AND[customer_address.ca_state IN ('IA', 'MS', 'WA'),(web_sales.ws_net_profit <= 250.00)]]) build RFs:RF7 ca_address_sk->wr_refunded_addr_sk ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((cd1.cd_education_status = cd2.cd_education_status) and (cd1.cd_marital_status = cd2.cd_marital_status) and (cd2.cd_demo_sk = web_returns.wr_returning_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->wr_returning_cdemo_sk;RF5 cd_marital_status->cd_marital_status;RF6 cd_education_status->cd_education_status --------------------------------PhysicalProject @@ -30,13 +30,13 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------PhysicalOlapScan[web_page] ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Secondary')]] and cd_education_status IN ('4 yr Degree', 'College', 'Secondary') and cd_marital_status IN ('D', 'M', 'S')) +--------------------------------------filter(OR[AND[(cd1.cd_marital_status = 'M'),(cd1.cd_education_status = '4 yr Degree')],AND[(cd1.cd_marital_status = 'S'),(cd1.cd_education_status = 'College')],AND[(cd1.cd_marital_status = 'D'),(cd1.cd_education_status = 'Secondary')]] and cd1.cd_education_status IN ('4 yr Degree', 'College', 'Secondary') and cd1.cd_marital_status IN ('D', 'M', 'S')) ----------------------------------------PhysicalOlapScan[customer_demographics(cd1)] apply RFs: RF5 RF6 --------------------------------PhysicalProject -----------------------------------filter(cd_education_status IN ('4 yr Degree', 'College', 'Secondary') and cd_marital_status IN ('D', 'M', 'S')) +----------------------------------filter(cd2.cd_education_status IN ('4 yr Degree', 'College', 'Secondary') and cd2.cd_marital_status IN ('D', 'M', 'S')) ------------------------------------PhysicalOlapScan[customer_demographics(cd2)] ----------------------------PhysicalProject -------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('AR', 'CA', 'IA', 'MO', 'MS', 'NE', 'TX', 'VA', 'WA')) +------------------------------filter((customer_address.ca_country = 'United States') and customer_address.ca_state IN ('AR', 'CA', 'IA', 'MO', 'MS', 'NE', 'TX', 'VA', 'WA')) --------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------filter((date_dim.d_year = 2001)) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query88.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query88.out index 101bad617b4c4b..cf188cec032c0f 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query88.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query88.out @@ -20,7 +20,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF21 RF22 RF23 ----------------------------------PhysicalProject -------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4)) +------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (2, 3, 4)) --------------------------------------PhysicalOlapScan[household_demographics] ------------------------------PhysicalProject --------------------------------filter((time_dim.t_hour = 8) and (time_dim.t_minute >= 30)) @@ -40,7 +40,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF18 RF19 RF20 ----------------------------------PhysicalProject -------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4)) +------------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (2, 3, 4)) --------------------------------------PhysicalOlapScan[household_demographics] ------------------------------PhysicalProject --------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute < 30)) @@ -60,7 +60,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF15 RF16 RF17 --------------------------------PhysicalProject -----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4)) +----------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (2, 3, 4)) ------------------------------------PhysicalOlapScan[household_demographics] ----------------------------PhysicalProject ------------------------------filter((time_dim.t_hour = 9) and (time_dim.t_minute >= 30)) @@ -80,7 +80,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 RF13 RF14 ------------------------------PhysicalProject ---------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4)) +--------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (2, 3, 4)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalProject ----------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute < 30)) @@ -100,7 +100,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF9 RF10 RF11 ----------------------------PhysicalProject -------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4)) +------------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (2, 3, 4)) --------------------------------PhysicalOlapScan[household_demographics] ------------------------PhysicalProject --------------------------filter((time_dim.t_hour = 10) and (time_dim.t_minute >= 30)) @@ -120,7 +120,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF6 RF7 RF8 --------------------------PhysicalProject -----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4)) +----------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (2, 3, 4)) ------------------------------PhysicalOlapScan[household_demographics] ----------------------PhysicalProject ------------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute < 30)) @@ -140,7 +140,7 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[store_sales] apply RFs: RF3 RF4 RF5 ------------------------PhysicalProject ---------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4)) +--------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (2, 3, 4)) ----------------------------PhysicalOlapScan[household_demographics] --------------------PhysicalProject ----------------------filter((time_dim.t_hour = 11) and (time_dim.t_minute >= 30)) @@ -160,7 +160,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ----------------------PhysicalProject -------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and hd_dep_count IN (2, 3, 4)) +------------------------filter((household_demographics.hd_vehicle_count <= 6) and OR[AND[(household_demographics.hd_dep_count = 2),(household_demographics.hd_vehicle_count <= 4)],(household_demographics.hd_dep_count = 4),AND[(household_demographics.hd_dep_count = 3),(household_demographics.hd_vehicle_count <= 5)]] and household_demographics.hd_dep_count IN (2, 3, 4)) --------------------------PhysicalOlapScan[household_demographics] ------------------PhysicalProject --------------------filter((time_dim.t_hour = 12) and (time_dim.t_minute < 30)) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query89.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query89.out index 2b0071500aec16..2aa648e6e42796 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query89.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query89.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------PhysicalTopN[LOCAL_SORT] ----------PhysicalProject -------------filter(( not (avg_monthly_sales = 0.0000)) and ((cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) +------------filter(( not (tmp1.avg_monthly_sales = 0.0000)) and ((cast(abs((tmp1.sum_sales - cast(tmp1.avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / tmp1.avg_monthly_sales) > 0.100000)) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------hashAgg[GLOBAL] @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject ---------------------------------------filter(OR[AND[i_category IN ('Books', 'Home', 'Music'),i_class IN ('classical', 'fiction', 'glassware')],AND[i_category IN ('Jewelry', 'Sports', 'Women'),i_class IN ('baseball', 'dresses', 'semi-precious')]] and i_category IN ('Books', 'Home', 'Jewelry', 'Music', 'Sports', 'Women') and i_class IN ('baseball', 'classical', 'dresses', 'fiction', 'glassware', 'semi-precious')) +--------------------------------------filter(OR[AND[item.i_category IN ('Books', 'Home', 'Music'),item.i_class IN ('classical', 'fiction', 'glassware')],AND[item.i_category IN ('Jewelry', 'Sports', 'Women'),item.i_class IN ('baseball', 'dresses', 'semi-precious')]] and item.i_category IN ('Books', 'Home', 'Jewelry', 'Music', 'Sports', 'Women') and item.i_class IN ('baseball', 'classical', 'dresses', 'fiction', 'glassware', 'semi-precious')) ----------------------------------------PhysicalOlapScan[item] --------------------------------PhysicalProject ----------------------------------filter((date_dim.d_year = 2000)) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query91.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query91.out index 690086a946d581..c951efff385f75 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query91.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query91.out @@ -33,9 +33,9 @@ PhysicalResultSink ------------------------------filter((customer_address.ca_gmt_offset = -7.00)) --------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject ---------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and cd_education_status IN ('Advanced Degree', 'Unknown') and cd_marital_status IN ('M', 'W')) +--------------------------filter(OR[AND[(customer_demographics.cd_marital_status = 'M'),(customer_demographics.cd_education_status = 'Unknown')],AND[(customer_demographics.cd_marital_status = 'W'),(customer_demographics.cd_education_status = 'Advanced Degree')]] and customer_demographics.cd_education_status IN ('Advanced Degree', 'Unknown') and customer_demographics.cd_marital_status IN ('M', 'W')) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject -----------------------filter((hd_buy_potential like 'Unknown%')) +----------------------filter((household_demographics.hd_buy_potential like 'Unknown%')) ------------------------PhysicalOlapScan[household_demographics] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query92.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query92.out index bf0e6ddd1ca8ce..2fc88b86959c65 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query92.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query92.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) +------------filter((cast(web_sales.ws_ext_discount_amt as DECIMALV3(38, 5)) > (1.3 * avg(ws_ext_discount_amt) OVER(PARTITION BY i_item_sk)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query94.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query94.out index da4c1eb80ceac5..ba778eeca79868 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query94.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query94.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------hashAgg[LOCAL] ----------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((ws1.ws_order_number = wr1.wr_order_number)) otherCondition=() ------------------PhysicalProject ---------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF3 ws_order_number->ws_order_number +--------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF3 ws_order_number->ws_order_number ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF2 web_site_sk->ws_web_site_sk --------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query95.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query95.out index 31413749d5e54d..b725fd3e8019e8 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query95.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query95.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws_warehouse_sk = ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number +------hashJoin[INNER_JOIN shuffle] hashCondition=((ws1.ws_order_number = ws2.ws_order_number)) otherCondition=(( not (ws1.ws_warehouse_sk = ws2.ws_warehouse_sk))) build RFs:RF0 ws_order_number->ws_order_number --------PhysicalProject ----------PhysicalOlapScan[web_sales(ws1)] apply RFs: RF0 RF8 --------PhysicalProject diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query98.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query98.out index 079b62a5c091fb..7594f9fb0a71fe 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query98.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query98.out @@ -17,7 +17,7 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ----------------------------PhysicalProject -------------------------------filter(i_category IN ('Home', 'Men', 'Sports')) +------------------------------filter(item.i_category IN ('Home', 'Men', 'Sports')) --------------------------------PhysicalOlapScan[item] ------------------------PhysicalProject --------------------------filter((date_dim.d_date <= '2002-02-04') and (date_dim.d_date >= '2002-01-05')) diff --git a/regression-test/data/shape_check/tpch_sf1000/check_point/probeShortcutFactor.out b/regression-test/data/shape_check/tpch_sf1000/check_point/probeShortcutFactor.out index e279e5efc963c5..1930900f2ff50d 100644 --- a/regression-test/data/shape_check/tpch_sf1000/check_point/probeShortcutFactor.out +++ b/regression-test/data/shape_check/tpch_sf1000/check_point/probeShortcutFactor.out @@ -5,6 +5,6 @@ PhysicalResultSink ----PhysicalProject ------PhysicalOlapScan[orders] apply RFs: RF0 ----PhysicalProject -------filter(substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------filter(substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q12.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q12.out index a6f742c6586fd4..e32f7dc797b1ac 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q12.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q12.out @@ -12,6 +12,6 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[orders] apply RFs: RF0 ------------------PhysicalProject ---------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and l_shipmode IN ('MAIL', 'SHIP')) +--------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and lineitem.l_shipmode IN ('MAIL', 'SHIP')) ----------------------PhysicalOlapScan[lineitem] diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q13.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q13.out index a9c26e203f3c44..769f82fa566037 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q13.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q13.out @@ -17,6 +17,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(( not (o_comment like '%special%requests%'))) +------------------------------filter(( not (orders.o_comment like '%special%requests%'))) --------------------------------PhysicalOlapScan[orders] diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q16.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q16.out index 0af2db9e4453c8..3e7269700d7dca 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q16.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q16.out @@ -14,9 +14,9 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 ----------------------PhysicalProject -------------------------filter(( not (p_brand = 'Brand#45')) and ( not (p_type like 'MEDIUM POLISHED%')) and p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) +------------------------filter(( not (part.p_brand = 'Brand#45')) and ( not (part.p_type like 'MEDIUM POLISHED%')) and part.p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) --------------------------PhysicalOlapScan[part] ------------------PhysicalProject ---------------------filter((s_comment like '%Customer%Complaints%')) +--------------------filter((supplier.s_comment like '%Customer%Complaints%')) ----------------------PhysicalOlapScan[supplier] diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q17.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q17.out index 7c8c121cda9356..e01de3db947e76 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q17.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q17.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) +------------filter((cast(lineitem.l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q19.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q19.out index c2833e61548efa..bd97e9b33fcd68 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q19.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q19.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey +----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey ------------PhysicalProject ---------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR')) +--------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and lineitem.l_shipmode IN ('AIR REG', 'AIR')) ----------------PhysicalOlapScan[lineitem] apply RFs: RF0 ------------PhysicalProject ---------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) +--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and part.p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) ----------------PhysicalOlapScan[part] diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q2.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q2.out index 94ef196009f5d8..757d0d13d0641d 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q2.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q2.out @@ -28,6 +28,6 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 ------------------------------PhysicalProject ---------------------------------filter((p_type like '%BRASS') and (part.p_size = 15)) +--------------------------------filter((part.p_size = 15) and (part.p_type like '%BRASS')) ----------------------------------PhysicalLazyMaterializeOlapScan[part lazySlots:(part.p_mfgr)] diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q20-rewrite.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q20-rewrite.out index d6c99416edb47b..debb122499d930 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q20-rewrite.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q20-rewrite.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->ps_suppkey;RF5 s_suppkey->l_suppkey ------------PhysicalProject ---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(t1.ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] @@ -18,7 +18,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[partsupp] apply RFs: RF1 RF4 ------------------PhysicalProject ---------------------filter((p_name like 'forest%')) +--------------------filter((part.p_name like 'forest%')) ----------------------PhysicalOlapScan[part] ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->s_nationkey diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q20.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q20.out index a6ddf238796b7c..3dd91006836173 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q20.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q20.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->ps_suppkey;RF5 s_suppkey->l_suppkey ------------PhysicalProject ---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(partsupp.ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] @@ -18,7 +18,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[partsupp] apply RFs: RF1 RF4 ------------------PhysicalProject ---------------------filter((p_name like 'forest%')) +--------------------filter((part.p_name like 'forest%')) ----------------------PhysicalOlapScan[part] ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->s_nationkey diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q21.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q21.out index 4e8310bda5be2c..92d05b3563d768 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q21.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q21.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF4 l_orderkey->l_orderkey +----------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l2.l_suppkey = l1.l_suppkey))) build RFs:RF4 l_orderkey->l_orderkey ------------------PhysicalProject --------------------PhysicalOlapScan[lineitem(l2)] apply RFs: RF4 ------------------PhysicalProject @@ -16,7 +16,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------filter((orders.o_orderstatus = 'F')) --------------------------PhysicalOlapScan[orders] apply RFs: RF3 -----------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF2 l_orderkey->l_orderkey +----------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l3.l_suppkey = l1.l_suppkey))) build RFs:RF2 l_orderkey->l_orderkey ------------------------PhysicalProject --------------------------filter((l3.l_receiptdate > l3.l_commitdate)) ----------------------------PhysicalOlapScan[lineitem(l3)] apply RFs: RF2 diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q22.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q22.out index 1f2e35b4ecfbd3..7650b849e4a78f 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q22.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q22.out @@ -12,14 +12,14 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[orders] apply RFs: RF0 ------------------PhysicalProject ---------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) +--------------------NestedLoopJoin[INNER_JOIN](cast(customer.c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) ----------------------PhysicalProject -------------------------filter(substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------filter(substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------PhysicalOlapScan[customer] ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecGather] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter((customer.c_acctbal > 0.00) and substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------------filter((customer.c_acctbal > 0.00) and substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q7.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q7.out index a1a826bb817f47..0dc3817882b1f7 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q7.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q7.out @@ -23,13 +23,13 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[supplier] apply RFs: RF1 ------------------------------PhysicalProject ---------------------------------filter(n_name IN ('FRANCE', 'GERMANY')) +--------------------------------filter(n1.n_name IN ('FRANCE', 'GERMANY')) ----------------------------------PhysicalOlapScan[nation(n1)] ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n2.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->c_nationkey ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------PhysicalProject -------------------------filter(n_name IN ('FRANCE', 'GERMANY')) +------------------------filter(n2.n_name IN ('FRANCE', 'GERMANY')) --------------------------PhysicalOlapScan[nation(n2)] diff --git a/regression-test/data/shape_check/tpch_sf1000/dphyper/q9.out b/regression-test/data/shape_check/tpch_sf1000/dphyper/q9.out index 1117aecec9c166..3b069b0461b5a8 100644 --- a/regression-test/data/shape_check/tpch_sf1000/dphyper/q9.out +++ b/regression-test/data/shape_check/tpch_sf1000/dphyper/q9.out @@ -20,7 +20,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[lineitem] apply RFs: RF1 RF3 RF5 RF7 ------------------------------PhysicalProject ---------------------------------filter((p_name like '%green%')) +--------------------------------filter((part.p_name like '%green%')) ----------------------------------PhysicalOlapScan[part] apply RFs: RF8 ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->s_nationkey diff --git a/regression-test/data/shape_check/tpch_sf1000/hint/q12.out b/regression-test/data/shape_check/tpch_sf1000/hint/q12.out index bd619416286f66..349580325bf082 100644 --- a/regression-test/data/shape_check/tpch_sf1000/hint/q12.out +++ b/regression-test/data/shape_check/tpch_sf1000/hint/q12.out @@ -12,7 +12,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[orders] ------------------PhysicalProject ---------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and l_shipmode IN ('MAIL', 'SHIP')) +--------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and lineitem.l_shipmode IN ('MAIL', 'SHIP')) ----------------------PhysicalOlapScan[lineitem] Hint log: diff --git a/regression-test/data/shape_check/tpch_sf1000/hint/q13.out b/regression-test/data/shape_check/tpch_sf1000/hint/q13.out index f838d963e0e0ae..c1c1869f40a66d 100644 --- a/regression-test/data/shape_check/tpch_sf1000/hint/q13.out +++ b/regression-test/data/shape_check/tpch_sf1000/hint/q13.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(( not (o_comment like '%special%requests%'))) +------------------------------filter(( not (orders.o_comment like '%special%requests%'))) --------------------------------PhysicalOlapScan[orders] ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpch_sf1000/hint/q17.out b/regression-test/data/shape_check/tpch_sf1000/hint/q17.out index 7846d2f3b5e569..326bd6dd66c417 100644 --- a/regression-test/data/shape_check/tpch_sf1000/hint/q17.out +++ b/regression-test/data/shape_check/tpch_sf1000/hint/q17.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) +------------filter((cast(lineitem.l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpch_sf1000/hint/q19.out b/regression-test/data/shape_check/tpch_sf1000/hint/q19.out index 8cfb81dcef67cb..18a6719d5e0628 100644 --- a/regression-test/data/shape_check/tpch_sf1000/hint/q19.out +++ b/regression-test/data/shape_check/tpch_sf1000/hint/q19.out @@ -5,12 +5,12 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) +----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) ------------PhysicalProject ---------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR')) +--------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and lineitem.l_shipmode IN ('AIR REG', 'AIR')) ----------------PhysicalOlapScan[lineitem] ------------PhysicalProject ---------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) +--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and part.p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) ----------------PhysicalOlapScan[part] Hint log: diff --git a/regression-test/data/shape_check/tpch_sf1000/hint/q7.out b/regression-test/data/shape_check/tpch_sf1000/hint/q7.out index d546dd1883fde1..f65d197ab0b30b 100644 --- a/regression-test/data/shape_check/tpch_sf1000/hint/q7.out +++ b/regression-test/data/shape_check/tpch_sf1000/hint/q7.out @@ -19,7 +19,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[supplier] --------------------------PhysicalProject -----------------------------filter(n_name IN ('FRANCE', 'GERMANY')) +----------------------------filter(n1.n_name IN ('FRANCE', 'GERMANY')) ------------------------------PhysicalOlapScan[nation(n1)] ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() @@ -30,7 +30,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------PhysicalOlapScan[customer] --------------------------PhysicalProject -----------------------------filter(n_name IN ('FRANCE', 'GERMANY')) +----------------------------filter(n2.n_name IN ('FRANCE', 'GERMANY')) ------------------------------PhysicalOlapScan[nation(n2)] Hint log: diff --git a/regression-test/data/shape_check/tpch_sf1000/hint/q9.out b/regression-test/data/shape_check/tpch_sf1000/hint/q9.out index 31cdf1b6e1a3d9..560946734200cd 100644 --- a/regression-test/data/shape_check/tpch_sf1000/hint/q9.out +++ b/regression-test/data/shape_check/tpch_sf1000/hint/q9.out @@ -20,7 +20,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[lineitem] ------------------------------PhysicalProject ---------------------------------filter((p_name like '%green%')) +--------------------------------filter((part.p_name like '%green%')) ----------------------------------PhysicalOlapScan[part] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q12.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q12.out index a6f742c6586fd4..e32f7dc797b1ac 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q12.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q12.out @@ -12,6 +12,6 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[orders] apply RFs: RF0 ------------------PhysicalProject ---------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and l_shipmode IN ('MAIL', 'SHIP')) +--------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and lineitem.l_shipmode IN ('MAIL', 'SHIP')) ----------------------PhysicalOlapScan[lineitem] diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q13.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q13.out index 6165574e0ee265..a4e0fe63587a53 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q13.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q13.out @@ -12,7 +12,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() ----------------------PhysicalProject -------------------------filter(( not (o_comment like '%special%requests%'))) +------------------------filter(( not (orders.o_comment like '%special%requests%'))) --------------------------PhysicalOlapScan[orders] ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q16.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q16.out index 69b0387b20c9d9..4891ae18426ed7 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q16.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q16.out @@ -13,9 +13,9 @@ PhysicalResultSink --------------------PhysicalProject ----------------------PhysicalOlapScan[partsupp] apply RFs: RF0 --------------------PhysicalProject -----------------------filter((s_comment like '%Customer%Complaints%')) +----------------------filter((supplier.s_comment like '%Customer%Complaints%')) ------------------------PhysicalOlapScan[supplier] ------------------PhysicalProject ---------------------filter(( not (p_brand = 'Brand#45')) and ( not (p_type like 'MEDIUM POLISHED%')) and p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) +--------------------filter(( not (part.p_brand = 'Brand#45')) and ( not (part.p_type like 'MEDIUM POLISHED%')) and part.p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) ----------------------PhysicalOlapScan[part] diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q17.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q17.out index 1dc329977c7a8c..b29db30da9ef31 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q17.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q17.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) +------------filter((cast(lineitem.l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q19.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q19.out index c2833e61548efa..bd97e9b33fcd68 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q19.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q19.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey +----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey ------------PhysicalProject ---------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR')) +--------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and lineitem.l_shipmode IN ('AIR REG', 'AIR')) ----------------PhysicalOlapScan[lineitem] apply RFs: RF0 ------------PhysicalProject ---------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) +--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and part.p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) ----------------PhysicalOlapScan[part] diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q2.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q2.out index 6cb1f363e19337..541e5f8cf337b1 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q2.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q2.out @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 ------------------------------------PhysicalProject ---------------------------------------filter((p_type like '%BRASS') and (part.p_size = 15)) +--------------------------------------filter((part.p_size = 15) and (part.p_type like '%BRASS')) ----------------------------------------PhysicalLazyMaterializeOlapScan[part lazySlots:(part.p_mfgr)] --------------------------------PhysicalLazyMaterializeOlapScan[supplier lazySlots:(supplier.s_address,supplier.s_phone,supplier.s_comment)] apply RFs: RF2 ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q20-rewrite.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q20-rewrite.out index ce848ba5952f3f..38de3873f8c2b4 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q20-rewrite.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q20-rewrite.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->ps_suppkey;RF4 s_suppkey->l_suppkey ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF1 ps_partkey->l_partkey;RF2 ps_suppkey->l_suppkey +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(t1.ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF1 ps_partkey->l_partkey;RF2 ps_suppkey->l_suppkey --------------------PhysicalProject ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecHash] @@ -21,7 +21,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 RF3 ----------------------PhysicalProject -------------------------filter((p_name like 'forest%')) +------------------------filter((part.p_name like 'forest%')) --------------------------PhysicalOlapScan[part] ----------------PhysicalProject ------------------PhysicalOlapScan[supplier] apply RFs: RF5 diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q20.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q20.out index 58db1f9dcd16c3..77e40eaba16706 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q20.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q20.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->ps_suppkey;RF4 s_suppkey->l_suppkey ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF1 ps_partkey->l_partkey;RF2 ps_suppkey->l_suppkey +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(partsupp.ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF1 ps_partkey->l_partkey;RF2 ps_suppkey->l_suppkey --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] @@ -20,7 +20,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 RF3 ----------------------PhysicalProject -------------------------filter((p_name like 'forest%')) +------------------------filter((part.p_name like 'forest%')) --------------------------PhysicalOlapScan[part] ----------------PhysicalProject ------------------PhysicalOlapScan[supplier] apply RFs: RF5 diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q21.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q21.out index 8794927faf7ce5..12e73e01b1df1f 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q21.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q21.out @@ -13,10 +13,10 @@ PhysicalResultSink --------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = l1.l_orderkey)) otherCondition=() build RFs:RF3 o_orderkey->l_orderkey;RF4 o_orderkey->l_orderkey ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = l1.l_suppkey)) otherCondition=() build RFs:RF2 s_suppkey->l_suppkey ---------------------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF1 l_orderkey->l_orderkey +--------------------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l2.l_suppkey = l1.l_suppkey))) build RFs:RF1 l_orderkey->l_orderkey ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[lineitem(l2)] apply RFs: RF1 RF4 -----------------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF0 l_orderkey->l_orderkey +----------------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l3.l_suppkey = l1.l_suppkey))) build RFs:RF0 l_orderkey->l_orderkey ------------------------------PhysicalProject --------------------------------filter((l3.l_receiptdate > l3.l_commitdate)) ----------------------------------PhysicalOlapScan[lineitem(l3)] apply RFs: RF0 diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q22.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q22.out index 1f2e35b4ecfbd3..2757db2dd9db58 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q22.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q22.out @@ -14,12 +14,12 @@ PhysicalResultSink ------------------PhysicalProject --------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) ----------------------PhysicalProject -------------------------filter(substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------filter(substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------PhysicalOlapScan[customer] ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecGather] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter((customer.c_acctbal > 0.00) and substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------------filter((customer.c_acctbal > 0.00) and substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q7.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q7.out index bc43dd188b4821..d639d9f64a396d 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q7.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q7.out @@ -26,9 +26,9 @@ PhysicalResultSink ------------------------PhysicalOlapScan[customer] apply RFs: RF3 ------------------NestedLoopJoin[INNER_JOIN]OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]] --------------------PhysicalProject -----------------------filter(n_name IN ('FRANCE', 'GERMANY')) +----------------------filter(n1.n_name IN ('FRANCE', 'GERMANY')) ------------------------PhysicalOlapScan[nation(n1)] --------------------PhysicalProject -----------------------filter(n_name IN ('FRANCE', 'GERMANY')) +----------------------filter(n2.n_name IN ('FRANCE', 'GERMANY')) ------------------------PhysicalOlapScan[nation(n2)] diff --git a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q9.out b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q9.out index 1ec1f7b970d8d0..38a129b25f4463 100644 --- a/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q9.out +++ b/regression-test/data/shape_check/tpch_sf1000/nostats_rf_prune/q9.out @@ -20,7 +20,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[orders] --------------------------PhysicalProject -----------------------------filter((p_name like '%green%')) +----------------------------filter((part.p_name like '%green%')) ------------------------------PhysicalOlapScan[part] ----------------------PhysicalProject ------------------------PhysicalOlapScan[partsupp] diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q12.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q12.out index a6f742c6586fd4..e32f7dc797b1ac 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q12.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q12.out @@ -12,6 +12,6 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[orders] apply RFs: RF0 ------------------PhysicalProject ---------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and l_shipmode IN ('MAIL', 'SHIP')) +--------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and lineitem.l_shipmode IN ('MAIL', 'SHIP')) ----------------------PhysicalOlapScan[lineitem] diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q13.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q13.out index a9c26e203f3c44..769f82fa566037 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q13.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q13.out @@ -17,6 +17,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(( not (o_comment like '%special%requests%'))) +------------------------------filter(( not (orders.o_comment like '%special%requests%'))) --------------------------------PhysicalOlapScan[orders] diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q16.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q16.out index 0af2db9e4453c8..3e7269700d7dca 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q16.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q16.out @@ -14,9 +14,9 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 ----------------------PhysicalProject -------------------------filter(( not (p_brand = 'Brand#45')) and ( not (p_type like 'MEDIUM POLISHED%')) and p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) +------------------------filter(( not (part.p_brand = 'Brand#45')) and ( not (part.p_type like 'MEDIUM POLISHED%')) and part.p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) --------------------------PhysicalOlapScan[part] ------------------PhysicalProject ---------------------filter((s_comment like '%Customer%Complaints%')) +--------------------filter((supplier.s_comment like '%Customer%Complaints%')) ----------------------PhysicalOlapScan[supplier] diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q17.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q17.out index 7c8c121cda9356..e01de3db947e76 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q17.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q17.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) +------------filter((cast(lineitem.l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q19.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q19.out index c2833e61548efa..bd97e9b33fcd68 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q19.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q19.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey +----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey ------------PhysicalProject ---------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR')) +--------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and lineitem.l_shipmode IN ('AIR REG', 'AIR')) ----------------PhysicalOlapScan[lineitem] apply RFs: RF0 ------------PhysicalProject ---------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) +--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and part.p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) ----------------PhysicalOlapScan[part] diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q2.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q2.out index 94ef196009f5d8..757d0d13d0641d 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q2.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q2.out @@ -28,6 +28,6 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 ------------------------------PhysicalProject ---------------------------------filter((p_type like '%BRASS') and (part.p_size = 15)) +--------------------------------filter((part.p_size = 15) and (part.p_type like '%BRASS')) ----------------------------------PhysicalLazyMaterializeOlapScan[part lazySlots:(part.p_mfgr)] diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q20-rewrite.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q20-rewrite.out index d9946b3ce6dff6..b91b41bc18ff88 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q20-rewrite.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q20-rewrite.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->ps_suppkey;RF5 s_suppkey->l_suppkey ------------PhysicalProject ---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(t1.ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalDistribute[DistributionSpecHash] @@ -19,7 +19,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[partsupp] apply RFs: RF1 RF4 ------------------PhysicalProject ---------------------filter((p_name like 'forest%')) +--------------------filter((part.p_name like 'forest%')) ----------------------PhysicalOlapScan[part] ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->s_nationkey diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q20.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q20.out index a6ddf238796b7c..3dd91006836173 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q20.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q20.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->ps_suppkey;RF5 s_suppkey->l_suppkey ------------PhysicalProject ---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(partsupp.ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] @@ -18,7 +18,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[partsupp] apply RFs: RF1 RF4 ------------------PhysicalProject ---------------------filter((p_name like 'forest%')) +--------------------filter((part.p_name like 'forest%')) ----------------------PhysicalOlapScan[part] ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->s_nationkey diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q21.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q21.out index 48375971ef5ad6..488f6c3c22167e 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q21.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q21.out @@ -8,10 +8,10 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF4 l_orderkey->l_orderkey +----------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l2.l_suppkey = l1.l_suppkey))) build RFs:RF4 l_orderkey->l_orderkey ------------------PhysicalProject --------------------PhysicalOlapScan[lineitem(l2)] apply RFs: RF4 -------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF3 l_orderkey->l_orderkey +------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l3.l_suppkey = l1.l_suppkey))) build RFs:RF3 l_orderkey->l_orderkey --------------------PhysicalProject ----------------------filter((l3.l_receiptdate > l3.l_commitdate)) ------------------------PhysicalOlapScan[lineitem(l3)] apply RFs: RF3 diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q22.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q22.out index 1f2e35b4ecfbd3..2757db2dd9db58 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q22.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q22.out @@ -14,12 +14,12 @@ PhysicalResultSink ------------------PhysicalProject --------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) ----------------------PhysicalProject -------------------------filter(substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------filter(substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------PhysicalOlapScan[customer] ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecGather] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter((customer.c_acctbal > 0.00) and substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------------filter((customer.c_acctbal > 0.00) and substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q7.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q7.out index a1a826bb817f47..0dc3817882b1f7 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q7.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q7.out @@ -23,13 +23,13 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[supplier] apply RFs: RF1 ------------------------------PhysicalProject ---------------------------------filter(n_name IN ('FRANCE', 'GERMANY')) +--------------------------------filter(n1.n_name IN ('FRANCE', 'GERMANY')) ----------------------------------PhysicalOlapScan[nation(n1)] ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n2.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->c_nationkey ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------PhysicalProject -------------------------filter(n_name IN ('FRANCE', 'GERMANY')) +------------------------filter(n2.n_name IN ('FRANCE', 'GERMANY')) --------------------------PhysicalOlapScan[nation(n2)] diff --git a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q9.out b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q9.out index 580b1caf23aafd..134e4b82258ffa 100644 --- a/regression-test/data/shape_check/tpch_sf1000/rf_prune/q9.out +++ b/regression-test/data/shape_check/tpch_sf1000/rf_prune/q9.out @@ -20,7 +20,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[lineitem] apply RFs: RF1 RF3 RF5 RF7 ------------------------------PhysicalProject ---------------------------------filter((p_name like '%green%')) +--------------------------------filter((part.p_name like '%green%')) ----------------------------------PhysicalOlapScan[part] apply RFs: RF8 ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q12.out b/regression-test/data/shape_check/tpch_sf1000/shape/q12.out index a6f742c6586fd4..e32f7dc797b1ac 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q12.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q12.out @@ -12,6 +12,6 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[orders] apply RFs: RF0 ------------------PhysicalProject ---------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and l_shipmode IN ('MAIL', 'SHIP')) +--------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and lineitem.l_shipmode IN ('MAIL', 'SHIP')) ----------------------PhysicalOlapScan[lineitem] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q13.out b/regression-test/data/shape_check/tpch_sf1000/shape/q13.out index a9c26e203f3c44..769f82fa566037 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q13.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q13.out @@ -17,6 +17,6 @@ PhysicalResultSink ------------------------PhysicalDistribute[DistributionSpecHash] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter(( not (o_comment like '%special%requests%'))) +------------------------------filter(( not (orders.o_comment like '%special%requests%'))) --------------------------------PhysicalOlapScan[orders] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q16.out b/regression-test/data/shape_check/tpch_sf1000/shape/q16.out index 0af2db9e4453c8..3e7269700d7dca 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q16.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q16.out @@ -14,9 +14,9 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 ----------------------PhysicalProject -------------------------filter(( not (p_brand = 'Brand#45')) and ( not (p_type like 'MEDIUM POLISHED%')) and p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) +------------------------filter(( not (part.p_brand = 'Brand#45')) and ( not (part.p_type like 'MEDIUM POLISHED%')) and part.p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) --------------------------PhysicalOlapScan[part] ------------------PhysicalProject ---------------------filter((s_comment like '%Customer%Complaints%')) +--------------------filter((supplier.s_comment like '%Customer%Complaints%')) ----------------------PhysicalOlapScan[supplier] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q17.out b/regression-test/data/shape_check/tpch_sf1000/shape/q17.out index 7c8c121cda9356..e01de3db947e76 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q17.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q17.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) +------------filter((cast(lineitem.l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q19.out b/regression-test/data/shape_check/tpch_sf1000/shape/q19.out index c2833e61548efa..bd97e9b33fcd68 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q19.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q19.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey +----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey ------------PhysicalProject ---------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR')) +--------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and lineitem.l_shipmode IN ('AIR REG', 'AIR')) ----------------PhysicalOlapScan[lineitem] apply RFs: RF0 ------------PhysicalProject ---------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) +--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and part.p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) ----------------PhysicalOlapScan[part] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q2.out b/regression-test/data/shape_check/tpch_sf1000/shape/q2.out index 94ef196009f5d8..757d0d13d0641d 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q2.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q2.out @@ -28,6 +28,6 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 ------------------------------PhysicalProject ---------------------------------filter((p_type like '%BRASS') and (part.p_size = 15)) +--------------------------------filter((part.p_size = 15) and (part.p_type like '%BRASS')) ----------------------------------PhysicalLazyMaterializeOlapScan[part lazySlots:(part.p_mfgr)] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q20-rewrite.out b/regression-test/data/shape_check/tpch_sf1000/shape/q20-rewrite.out index d9946b3ce6dff6..b91b41bc18ff88 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q20-rewrite.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q20-rewrite.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->ps_suppkey;RF5 s_suppkey->l_suppkey ------------PhysicalProject ---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(t1.ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalDistribute[DistributionSpecHash] @@ -19,7 +19,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[partsupp] apply RFs: RF1 RF4 ------------------PhysicalProject ---------------------filter((p_name like 'forest%')) +--------------------filter((part.p_name like 'forest%')) ----------------------PhysicalOlapScan[part] ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->s_nationkey diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q20.out b/regression-test/data/shape_check/tpch_sf1000/shape/q20.out index a6ddf238796b7c..3dd91006836173 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q20.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q20.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->ps_suppkey;RF5 s_suppkey->l_suppkey ------------PhysicalProject ---------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(partsupp.ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->l_partkey;RF3 ps_suppkey->l_suppkey ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] @@ -18,7 +18,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[partsupp] apply RFs: RF1 RF4 ------------------PhysicalProject ---------------------filter((p_name like 'forest%')) +--------------------filter((part.p_name like 'forest%')) ----------------------PhysicalOlapScan[part] ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->s_nationkey diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q21.out b/regression-test/data/shape_check/tpch_sf1000/shape/q21.out index 48375971ef5ad6..488f6c3c22167e 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q21.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q21.out @@ -8,10 +8,10 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF4 l_orderkey->l_orderkey +----------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l2.l_suppkey = l1.l_suppkey))) build RFs:RF4 l_orderkey->l_orderkey ------------------PhysicalProject --------------------PhysicalOlapScan[lineitem(l2)] apply RFs: RF4 -------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF3 l_orderkey->l_orderkey +------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l3.l_suppkey = l1.l_suppkey))) build RFs:RF3 l_orderkey->l_orderkey --------------------PhysicalProject ----------------------filter((l3.l_receiptdate > l3.l_commitdate)) ------------------------PhysicalOlapScan[lineitem(l3)] apply RFs: RF3 diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q22.out b/regression-test/data/shape_check/tpch_sf1000/shape/q22.out index 1f2e35b4ecfbd3..2757db2dd9db58 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q22.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q22.out @@ -14,12 +14,12 @@ PhysicalResultSink ------------------PhysicalProject --------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) ----------------------PhysicalProject -------------------------filter(substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------filter(substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------PhysicalOlapScan[customer] ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecGather] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter((customer.c_acctbal > 0.00) and substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------------filter((customer.c_acctbal > 0.00) and substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q7.out b/regression-test/data/shape_check/tpch_sf1000/shape/q7.out index a1a826bb817f47..0dc3817882b1f7 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q7.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q7.out @@ -23,13 +23,13 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[supplier] apply RFs: RF1 ------------------------------PhysicalProject ---------------------------------filter(n_name IN ('FRANCE', 'GERMANY')) +--------------------------------filter(n1.n_name IN ('FRANCE', 'GERMANY')) ----------------------------------PhysicalOlapScan[nation(n1)] ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n2.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->c_nationkey ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------PhysicalProject -------------------------filter(n_name IN ('FRANCE', 'GERMANY')) +------------------------filter(n2.n_name IN ('FRANCE', 'GERMANY')) --------------------------PhysicalOlapScan[nation(n2)] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape/q9.out b/regression-test/data/shape_check/tpch_sf1000/shape/q9.out index 1117aecec9c166..3b069b0461b5a8 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape/q9.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape/q9.out @@ -20,7 +20,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[lineitem] apply RFs: RF1 RF3 RF5 RF7 ------------------------------PhysicalProject ---------------------------------filter((p_name like '%green%')) +--------------------------------filter((part.p_name like '%green%')) ----------------------------------PhysicalOlapScan[part] apply RFs: RF8 ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->s_nationkey diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q12.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q12.out index a6f742c6586fd4..e32f7dc797b1ac 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q12.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q12.out @@ -12,6 +12,6 @@ PhysicalResultSink ------------------PhysicalProject --------------------PhysicalOlapScan[orders] apply RFs: RF0 ------------------PhysicalProject ---------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and l_shipmode IN ('MAIL', 'SHIP')) +--------------------filter((lineitem.l_commitdate < lineitem.l_receiptdate) and (lineitem.l_receiptdate < '1995-01-01') and (lineitem.l_receiptdate >= '1994-01-01') and (lineitem.l_shipdate < '1995-01-01') and (lineitem.l_shipdate < lineitem.l_commitdate) and lineitem.l_shipmode IN ('MAIL', 'SHIP')) ----------------------PhysicalOlapScan[lineitem] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q13.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q13.out index 26e8f8b68fd615..f0368e6d3446fc 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q13.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q13.out @@ -12,7 +12,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->o_custkey ----------------------PhysicalProject -------------------------filter(( not (o_comment like '%special%requests%'))) +------------------------filter(( not (orders.o_comment like '%special%requests%'))) --------------------------PhysicalOlapScan[orders] apply RFs: RF0 ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q16.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q16.out index 69b0387b20c9d9..4891ae18426ed7 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q16.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q16.out @@ -13,9 +13,9 @@ PhysicalResultSink --------------------PhysicalProject ----------------------PhysicalOlapScan[partsupp] apply RFs: RF0 --------------------PhysicalProject -----------------------filter((s_comment like '%Customer%Complaints%')) +----------------------filter((supplier.s_comment like '%Customer%Complaints%')) ------------------------PhysicalOlapScan[supplier] ------------------PhysicalProject ---------------------filter(( not (p_brand = 'Brand#45')) and ( not (p_type like 'MEDIUM POLISHED%')) and p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) +--------------------filter(( not (part.p_brand = 'Brand#45')) and ( not (part.p_type like 'MEDIUM POLISHED%')) and part.p_size IN (14, 19, 23, 3, 36, 45, 49, 9)) ----------------------PhysicalOlapScan[part] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q17.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q17.out index 1dc329977c7a8c..b29db30da9ef31 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q17.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q17.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------filter((cast(l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) +------------filter((cast(lineitem.l_quantity as DECIMALV3(38, 5)) < (0.2 * avg(l_quantity) OVER(PARTITION BY p_partkey)))) --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q19.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q19.out index c2833e61548efa..bd97e9b33fcd68 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q19.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q19.out @@ -5,11 +5,11 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey +----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(lineitem.l_quantity <= 11.00),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(lineitem.l_quantity >= 10.00),(lineitem.l_quantity <= 20.00),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG'),(lineitem.l_quantity >= 20.00)]]) build RFs:RF0 p_partkey->l_partkey ------------PhysicalProject ---------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR')) +--------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and lineitem.l_shipmode IN ('AIR REG', 'AIR')) ----------------PhysicalOlapScan[lineitem] apply RFs: RF0 ------------PhysicalProject ---------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) +--------------filter((part.p_size <= 15) and (part.p_size >= 1) and OR[AND[(part.p_brand = 'Brand#12'),part.p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG'),(part.p_size <= 5)],AND[(part.p_brand = 'Brand#23'),part.p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG'),(part.p_size <= 10)],AND[(part.p_brand = 'Brand#34'),part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')]] and part.p_brand IN ('Brand#12', 'Brand#23', 'Brand#34') and part.p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG', 'MED BAG', 'MED BOX', 'MED PACK', 'MED PKG', 'SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) ----------------PhysicalOlapScan[part] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q2.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q2.out index 99f729cafccf96..13cb3f558f5937 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q2.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q2.out @@ -21,7 +21,7 @@ PhysicalResultSink ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 RF1 ------------------------------------PhysicalProject ---------------------------------------filter((p_type like '%BRASS') and (part.p_size = 15)) +--------------------------------------filter((part.p_size = 15) and (part.p_type like '%BRASS')) ----------------------------------------PhysicalLazyMaterializeOlapScan[part lazySlots:(part.p_mfgr)] --------------------------------PhysicalLazyMaterializeOlapScan[supplier lazySlots:(supplier.s_address,supplier.s_phone,supplier.s_comment)] apply RFs: RF2 ----------------------------PhysicalProject diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q20-rewrite.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q20-rewrite.out index ce848ba5952f3f..38de3873f8c2b4 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q20-rewrite.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q20-rewrite.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->ps_suppkey;RF4 s_suppkey->l_suppkey ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF1 ps_partkey->l_partkey;RF2 ps_suppkey->l_suppkey +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(t1.ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF1 ps_partkey->l_partkey;RF2 ps_suppkey->l_suppkey --------------------PhysicalProject ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecHash] @@ -21,7 +21,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 RF3 ----------------------PhysicalProject -------------------------filter((p_name like 'forest%')) +------------------------filter((part.p_name like 'forest%')) --------------------------PhysicalOlapScan[part] ----------------PhysicalProject ------------------PhysicalOlapScan[supplier] apply RFs: RF5 diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q20.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q20.out index 58db1f9dcd16c3..77e40eaba16706 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q20.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q20.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalProject --------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->ps_suppkey;RF4 s_suppkey->l_suppkey ----------------PhysicalProject -------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF1 ps_partkey->l_partkey;RF2 ps_suppkey->l_suppkey +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(partsupp.ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF1 ps_partkey->l_partkey;RF2 ps_suppkey->l_suppkey --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] @@ -20,7 +20,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 RF3 ----------------------PhysicalProject -------------------------filter((p_name like 'forest%')) +------------------------filter((part.p_name like 'forest%')) --------------------------PhysicalOlapScan[part] ----------------PhysicalProject ------------------PhysicalOlapScan[supplier] apply RFs: RF5 diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q21.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q21.out index 8794927faf7ce5..12e73e01b1df1f 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q21.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q21.out @@ -13,10 +13,10 @@ PhysicalResultSink --------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = l1.l_orderkey)) otherCondition=() build RFs:RF3 o_orderkey->l_orderkey;RF4 o_orderkey->l_orderkey ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = l1.l_suppkey)) otherCondition=() build RFs:RF2 s_suppkey->l_suppkey ---------------------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF1 l_orderkey->l_orderkey +--------------------------hashJoin[RIGHT_SEMI_JOIN colocated] hashCondition=((l2.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l2.l_suppkey = l1.l_suppkey))) build RFs:RF1 l_orderkey->l_orderkey ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[lineitem(l2)] apply RFs: RF1 RF4 -----------------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l_suppkey = l_suppkey))) build RFs:RF0 l_orderkey->l_orderkey +----------------------------hashJoin[RIGHT_ANTI_JOIN colocated] hashCondition=((l3.l_orderkey = l1.l_orderkey)) otherCondition=(( not (l3.l_suppkey = l1.l_suppkey))) build RFs:RF0 l_orderkey->l_orderkey ------------------------------PhysicalProject --------------------------------filter((l3.l_receiptdate > l3.l_commitdate)) ----------------------------------PhysicalOlapScan[lineitem(l3)] apply RFs: RF0 diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q22.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q22.out index 1f2e35b4ecfbd3..2757db2dd9db58 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q22.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q22.out @@ -14,12 +14,12 @@ PhysicalResultSink ------------------PhysicalProject --------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal)) ----------------------PhysicalProject -------------------------filter(substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------filter(substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------PhysicalOlapScan[customer] ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecGather] --------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------filter((customer.c_acctbal > 0.00) and substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +------------------------------filter((customer.c_acctbal > 0.00) and substring(customer.c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q7.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q7.out index bc43dd188b4821..d639d9f64a396d 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q7.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q7.out @@ -26,9 +26,9 @@ PhysicalResultSink ------------------------PhysicalOlapScan[customer] apply RFs: RF3 ------------------NestedLoopJoin[INNER_JOIN]OR[AND[(n1.n_name = 'FRANCE'),(n2.n_name = 'GERMANY')],AND[(n1.n_name = 'GERMANY'),(n2.n_name = 'FRANCE')]] --------------------PhysicalProject -----------------------filter(n_name IN ('FRANCE', 'GERMANY')) +----------------------filter(n1.n_name IN ('FRANCE', 'GERMANY')) ------------------------PhysicalOlapScan[nation(n1)] --------------------PhysicalProject -----------------------filter(n_name IN ('FRANCE', 'GERMANY')) +----------------------filter(n2.n_name IN ('FRANCE', 'GERMANY')) ------------------------PhysicalOlapScan[nation(n2)] diff --git a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q9.out b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q9.out index 067e5ab1984e6d..940bbd4f8ae8b4 100644 --- a/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q9.out +++ b/regression-test/data/shape_check/tpch_sf1000/shape_no_stats/q9.out @@ -20,7 +20,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[orders] --------------------------PhysicalProject -----------------------------filter((p_name like '%green%')) +----------------------------filter((part.p_name like '%green%')) ------------------------------PhysicalOlapScan[part] apply RFs: RF5 ----------------------PhysicalProject ------------------------PhysicalOlapScan[partsupp] apply RFs: RF7 diff --git a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out index b0efcf1d5b6ee1..ba759342923fc8 100644 --- a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out +++ b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out @@ -1,7 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !show -- +-- !show_initial_replica_1 -- show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"binlog.format" = "STATEMENT_AND_SNAPSHOT",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +-- !show_initial_replica_3 -- +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"binlog.format" = "STATEMENT_AND_SNAPSHOT",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + -- !select -- 1 1 30 1 2 5 @@ -35,11 +38,20 @@ show_create_table_and_views_nereids_view CREATE VIEW `show_create_table_and_view 200 1 300 1 --- !show -- +-- !show_after_rollup_replica_1 -- show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"binlog.format" = "STATEMENT_AND_SNAPSHOT",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); --- !show -- +-- !show_after_rollup_replica_3 -- +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"binlog.format" = "STATEMENT_AND_SNAPSHOT",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !show_like_replica_1 -- show_create_table_and_views_nereids_like CREATE TABLE `show_create_table_and_views_nereids_like` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"binlog.format" = "STATEMENT_AND_SNAPSHOT",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); --- !show -- +-- !show_like_replica_3 -- +show_create_table_and_views_nereids_like CREATE TABLE `show_create_table_and_views_nereids_like` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"binlog.format" = "STATEMENT_AND_SNAPSHOT",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !show_like_with_rollup_replica_1 -- show_create_table_and_views_nereids_like_with_rollup CREATE TABLE `show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"binlog.format" = "STATEMENT_AND_SNAPSHOT",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !show_like_with_rollup_replica_3 -- +show_create_table_and_views_nereids_like_with_rollup CREATE TABLE `show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` largeint NOT NULL,\n `good_id` largeint NOT NULL,\n `cost` bigint SUM NULL DEFAULT "0",\n INDEX index_user_id (`user_id`) USING INVERTED COMMENT "test index comment",\n INDEX index_good_id (`good_id`) USING INVERTED COMMENT "test index\\" comment"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V3",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"binlog.format" = "STATEMENT_AND_SNAPSHOT",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); diff --git a/regression-test/data/table_stream_p0/test_mow_min_delta_delete_before.out b/regression-test/data/table_stream_p0/test_mow_min_delta_delete_before.out new file mode 100644 index 00000000000000..511584038f6ada --- /dev/null +++ b/regression-test/data/table_stream_p0/test_mow_min_delta_delete_before.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mow_min_delta_delete_before -- +1 10 DELETE + +-- !mow_min_delta_del_ins_del -- +2 20 DELETE + diff --git a/regression-test/data/table_stream_p0/test_olap_table_stream_mixed_history_incremental.out b/regression-test/data/table_stream_p0/test_olap_table_stream_mixed_history_incremental.out index 8875bf944a5e64..0011e50ed8b010 100644 --- a/regression-test/data/table_stream_p0/test_olap_table_stream_mixed_history_incremental.out +++ b/regression-test/data/table_stream_p0/test_olap_table_stream_mixed_history_incremental.out @@ -6,3 +6,4 @@ -- !mixed_history_incremental_count -- 1 1 11 1 + diff --git a/regression-test/data/table_stream_p0/test_table_stream_query_comprehensive.out b/regression-test/data/table_stream_p0/test_table_stream_query_comprehensive.out index 65645ba2bde3e0..e72e41e5b9089e 100644 --- a/regression-test/data/table_stream_p0/test_table_stream_query_comprehensive.out +++ b/regression-test/data/table_stream_p0/test_table_stream_query_comprehensive.out @@ -36,3 +36,4 @@ 2 p2b 3 p3 3 p3b + diff --git a/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out b/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out new file mode 100644 index 00000000000000..505edfbb5460f5 --- /dev/null +++ b/regression-test/data/variant_p0/test_topn_lazy_materialize_sparse_variant.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !topn_lazy_sparse_variant -- +0 1 {} {} +1 -1 {} [1,2,3] +3 42 {"k":1} {} +4 -9223372036854775808 [1,2,3] {"k":1} +6 1 [1,2,3] {} +7 0 [1,2,3] [1,2,3] +9 \N {} [] +10 1 {} {"n":7,"word":"hot"} +12 1 {"word":"hot","n":7} \N +13 5777142737451291289 {} {"n":7,"word":"hot"} +15 -1 [1,2,3] [] +16 -906638365906932436 {"word":"hot","n":7} {"k":1} +18 -9223372036854775808 {"word":"hot","n":7} {"n":7,"word":"hot"} +19 -9223372036854775808 {"word":"hot","n":7} [1,2,3] + diff --git a/regression-test/framework/pom.xml b/regression-test/framework/pom.xml index 45b589185b854b..52660d5bcff5a6 100644 --- a/regression-test/framework/pom.xml +++ b/regression-test/framework/pom.xml @@ -266,7 +266,7 @@ under the License. org.junit.jupiter junit-jupiter-api - 5.8.2 + 5.10.2 mysql diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 832669de082840..cedb00cc0e3e56 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -3753,35 +3753,35 @@ class Suite implements GroovyInterceptable { throw new IllegalArgumentException("invalid caseElapseSeconds, ${caseElapseSeconds}") } - long sleepSeconds = 0 - LocalDateTime now = LocalDateTime.now(); + LocalDateTime now = LocalDateTime.now() + LocalDateTime boundary switch (caseSpanConstraint) { case "NOT_CROSS_HOUR_BOUNDARY": - LocalDateTime nextHour = now.withMinute(0).withSecond(0).withNano(0).plusHours(1); - long secondsToNextHour = ChronoUnit.SECONDS.between(now, nextHour) - - if (secondsToNextHour < caseElapseSeconds) { - sleepSeconds = secondsToNextHour - } + boundary = now.withMinute(0).withSecond(0).withNano(0).plusHours(1) break case "NOT_CROSS_DAY_BOUNDARY": - LocalDateTime startOfNextDay = now.toLocalDate().plusDays(1).atStartOfDay(); - long secondsToNextDay = ChronoUnit.SECONDS.between(now, startOfNextDay) - - if (secondsToNextDay < caseElapseSeconds) { - sleepSeconds = secondsToNextDay - } + boundary = now.toLocalDate().plusDays(1).atStartOfDay() break default: throw new IllegalArgumentException("invalid caseSpanConstraint:${caseSpanConstraint}") } - if (sleepSeconds > 0) { - logger.info("test sleeps ${sleepSeconds} to satisfy ${caseSpanConstraint}") - Thread.sleep(sleepSeconds * 1000) + long sleepMillis = calculateBoundarySleepMillis(now, boundary, caseElapseSeconds) + if (sleepMillis > 0) { + logger.info("test sleeps ${sleepMillis} ms to satisfy ${caseSpanConstraint}") + Thread.sleep(sleepMillis) + } + } + + static long calculateBoundarySleepMillis(LocalDateTime now, LocalDateTime boundary, int caseElapseSeconds) { + long millisToBoundary = ChronoUnit.MILLIS.between(now, boundary) + if (millisToBoundary <= TimeUnit.SECONDS.toMillis(caseElapseSeconds)) { + // Cross the boundary by a full millisecond; truncated fractional milliseconds must not resume early. + return millisToBoundary + 1 } + return 0 } void retryUntilHasSqlCache(String sql) { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RemoteFileOperator.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RemoteFileOperator.groovy index 13f404f928dd2d..585b387bd0015e 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RemoteFileOperator.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/RemoteFileOperator.groovy @@ -100,7 +100,7 @@ class RemoteFileOperator { } if (execResult.exitCode != 0) { - def errorMsg = "Failed to create directory on ${host} (exit code: ${execResult.exitCode}): ${execResult.error}" + def errorMsg = "Failed to create directory on ${host} (exit code: ${execResult.exitCode}): ${execResult.stderr}" logger.error(errorMsg) throw new Exception(errorMsg) } @@ -140,7 +140,9 @@ class RemoteFileOperator { hosts.each { host -> // Step 1: Check if remote directory has files (count regular files only) // scp user@host:/tmp/doristest/* will fail if doristest has no files. - String countCommand = """ssh -p ${port} ${username}@${host} "find ${escapePath(remoteDir)} -maxdepth 1 -type f | wc -l" """ + String countCommand = isLocalHost(host) + ? "find ${escapePath(remoteDir)} -maxdepth 1 -type f | wc -l" + : """ssh -p ${port} ${username}@${host} "find ${escapePath(remoteDir)} -maxdepth 1 -type f | wc -l" """ def countResult = executeLocalCommand(countCommand) if (countResult.timedOut) { @@ -150,7 +152,7 @@ class RemoteFileOperator { } if (countResult.exitCode != 0) { - def errorMsg = "Failed to check file count on ${host} (exit code: ${countResult.exitCode}): ${countResult.error}" + def errorMsg = "Failed to check file count on ${host} (exit code: ${countResult.exitCode}): ${countResult.stderr}" logger.error(errorMsg) throw new Exception(errorMsg) } @@ -163,9 +165,16 @@ class RemoteFileOperator { if (fileCount > 0) { logger.info("Downloading ${fileCount} files from ${host}:${remoteDir} to ${localBaseDir}") - def normalizedRemoteDir = (remoteDir.endsWith('/') ? remoteDir : "${remoteDir}/") + "*" - def scpCommand = "scp -P ${port} ${username}@${host}:${escapePath(normalizedRemoteDir)} ${escapePath(localBaseDir)}" - def execResult = executeLocalCommand(scpCommand) + def copyCommand + if (isLocalHost(host)) { + copyCommand = "find ${escapePath(remoteDir)} -maxdepth 1 -type f " + + "-exec cp -f '{}' ${escapePath(localBaseDir)} ';'" + } else { + def normalizedRemoteDir = (remoteDir.endsWith('/') ? remoteDir : "${remoteDir}/") + "*" + copyCommand = "scp -P ${port} ${username}@${host}:${escapePath(normalizedRemoteDir)} " + + escapePath(localBaseDir) + } + def execResult = executeLocalCommand(copyCommand) if (execResult.timedOut) { def errorMsg = "Timeout downloading from ${host} (${timeout}ms)" @@ -174,7 +183,7 @@ class RemoteFileOperator { } if (execResult.exitCode != 0) { - def errorMsg = "Failed to download from ${host} (exit code: ${execResult.exitCode}): ${execResult.error}" + def errorMsg = "Failed to download from ${host} (exit code: ${execResult.exitCode}): ${execResult.stderr}" logger.error(errorMsg) throw new Exception(errorMsg) } @@ -214,7 +223,7 @@ class RemoteFileOperator { } if (execResult.exitCode != 0) { - def errorMsg = "Failed to delete directory on ${host} (exit code: ${execResult.exitCode}): ${execResult.error}" + def errorMsg = "Failed to delete directory on ${host} (exit code: ${execResult.exitCode}): ${execResult.stderr}" logger.error(errorMsg) throw new Exception(errorMsg) } @@ -224,10 +233,17 @@ class RemoteFileOperator { } private Map executeSshCommand(String host, String command) { + if (isLocalHost(host)) { + return executeLocalCommand(command) + } def sshCommand = "ssh -p ${port} ${username}@${host} '${command}'" return executeLocalCommand(sshCommand) } + private boolean isLocalHost(String host) { + return host in ["127.0.0.1", "localhost", "::1"] + } + private Map executeLocalCommand(String command) { def result = [ exitCode: -1, diff --git a/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy new file mode 100644 index 00000000000000..4b8693e7678d6a --- /dev/null +++ b/regression-test/framework/src/test/groovy/org/apache/doris/regression/suite/SuiteBoundaryWaitTest.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import org.junit.jupiter.api.Test + +import java.time.LocalDateTime +import java.util.concurrent.TimeUnit + +import static org.junit.jupiter.api.Assertions.assertEquals +import static org.junit.jupiter.api.Assertions.assertTrue + +class SuiteBoundaryWaitTest { + @Test + void waitCrossesBoundaryWhenRemainingTimeHasFractionalSecond() { + LocalDateTime now = LocalDateTime.parse("2026-07-16T22:59:40.285999999") + LocalDateTime nextHour = LocalDateTime.parse("2026-07-16T23:00:00") + + long waitMillis = Suite.calculateBoundarySleepMillis(now, nextHour, 45) + + assertTrue(now.plusNanos(TimeUnit.MILLISECONDS.toNanos(waitMillis)).isAfter(nextHour)) + } + + @Test + void waitCrossesBoundaryWhenExpectedDurationEndsExactlyAtBoundary() { + LocalDateTime now = LocalDateTime.parse("2026-07-16T22:59:15") + LocalDateTime nextHour = LocalDateTime.parse("2026-07-16T23:00:00") + + long waitMillis = Suite.calculateBoundarySleepMillis(now, nextHour, 45) + + assertTrue(now.plusNanos(TimeUnit.MILLISECONDS.toNanos(waitMillis)).isAfter(nextHour)) + } + + @Test + void noWaitWhenExpectedDurationFinishesBeforeBoundary() { + LocalDateTime now = LocalDateTime.parse("2026-07-16T22:59:14.999") + LocalDateTime nextHour = LocalDateTime.parse("2026-07-16T23:00:00") + + assertEquals(0, Suite.calculateBoundarySleepMillis(now, nextHour, 45)) + } +} diff --git a/regression-test/java-udf-src/pom.xml b/regression-test/java-udf-src/pom.xml index 388bf795e1ec4b..c8ab3ba71adcdc 100644 --- a/regression-test/java-udf-src/pom.xml +++ b/regression-test/java-udf-src/pom.xml @@ -26,8 +26,7 @@ under the License. Java UDF Case https://doris.apache.org/ - 8 - 8 + 17 3.1.3 @@ -85,10 +84,6 @@ under the License. org.apache.maven.plugins maven-compiler-plugin 3.10.1 - - 8 - 8 - diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy index 5cc507a9a14608..d6679434eccbd0 100644 --- a/regression-test/pipeline/external/conf/regression-conf.groovy +++ b/regression-test/pipeline/external/conf/regression-conf.groovy @@ -190,6 +190,7 @@ oracle_11_port=1521 sqlserver_2022_port=1433 clickhouse_22_port=8123 oceanbase_port=2881 +oceanbase_cdc_port=2883 db2_11_port=50000 // trino-connector catalog test config diff --git a/regression-test/suites/auth_call/test_ddl_restore_auth.groovy b/regression-test/suites/auth_call/test_ddl_restore_auth.groovy index 5079b7ce4163b8..288099b9b4fb4c 100644 --- a/regression-test/suites/auth_call/test_ddl_restore_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_restore_auth.groovy @@ -17,7 +17,7 @@ import java.util.UUID -suite("test_ddl_restore_auth", "p0,auth_call") { +suite("test_ddl_restore_auth", "p0,auth_call,nonConcurrent") { String label = UUID.randomUUID().toString().replaceAll("-", "").substring(0, 8) def syncer = getSyncer() diff --git a/regression-test/suites/cast_p0/cast_to_datetime.groovy b/regression-test/suites/cast_p0/cast_to_datetime.groovy index 7a3084bd8e4800..59960b54da28fc 100644 --- a/regression-test/suites/cast_p0/cast_to_datetime.groovy +++ b/regression-test/suites/cast_p0/cast_to_datetime.groovy @@ -237,6 +237,11 @@ qt_sql204 """ select cast('9999-12-31 23:59:59.999999 +00:00' as datetime(6)) "" testFoldConst("select cast('9999-12-31 23:59:59.999999 +00:00' as datetime(6))") qt_sql205 """ select cast('0000-01-01 00:00:00+12:00' as datetime(6)); """ testFoldConst("select cast('0000-01-01 00:00:00+12:00' as datetime(6));") +testFoldConst("select cast('9999-12-31 23:59:59.999999' as date), " + + "cast('9999-12-31 23:59:59.999999' as datev2)") +waitUntilSafeExecutionTime("NOT_CROSS_DAY_BOUNDARY", 2) +testFoldConst("select cast(cast('-00:00:01' as time(0)) as datetime(0)), " + + "cast(cast('-00:00:01' as time(0)) as datetimev2(0))") sql "set debug_skip_fold_constant = false" @@ -258,4 +263,4 @@ testFoldConst("select cast('0000-01-01 00:00:00+12:00' as datetime(6));") sql "select cast('0000-01-01 00:00:00+12:00' as datetime(6));" exception "out of range" } -} \ No newline at end of file +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load.groovy b/regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load.groovy index 6e702b96c29485..f2bec9bb3520d6 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/stream_load/stream_load.groovy @@ -312,6 +312,109 @@ suite("stream_load") { assertTrue(before_cluster0_load_rows < after_cluster0_load_rows) assertTrue(before_cluster0_flush < after_cluster0_flush) + assertTrue(before_cluster1_load_rows == after_cluster1_load_rows) + assertTrue(before_cluster1_flush == after_cluster1_flush) + + // case4 FE redirects HTTP stream by compute_group and planning follows the receiving backend + before_cluster0_load_rows = get_be_metric(ipList[0], httpPortList[0], "load_rows"); + log.info("before_cluster0_load_rows : ${before_cluster0_load_rows}".toString()) + before_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total"); + log.info("before_cluster0_flush : ${before_cluster0_flush}".toString()) + + before_cluster1_load_rows = get_be_metric(ipList[1], httpPortList[1], "load_rows"); + log.info("before_cluster1_load_rows : ${before_cluster1_load_rows}".toString()) + before_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total"); + log.info("before_cluster1_flush : ${before_cluster1_flush}".toString()) + + streamLoad { + set 'version', '1' + set 'sql', """ + insert into ${context.dbName}.${tableName4} + select * from http_stream("format"="csv", "column_separator"=",") + """ + set 'compute_group', 'stream_load_cluster_name1' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("HTTP stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + } + } + sql "sync" + order_qt_http_stream_compute_group "SELECT count(*) FROM ${tableName4}" + + after_cluster0_load_rows = get_be_metric(ipList[0], httpPortList[0], "load_rows"); + log.info("after_cluster0_load_rows : ${after_cluster0_load_rows}".toString()) + after_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total"); + log.info("after_cluster0_flush : ${after_cluster0_flush}".toString()) + + after_cluster1_load_rows = get_be_metric(ipList[1], httpPortList[1], "load_rows"); + log.info("after_cluster1_load_rows : ${after_cluster1_load_rows}".toString()) + after_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total"); + log.info("after_cluster1_flush : ${after_cluster1_flush}".toString()) + + assertTrue(before_cluster0_load_rows == after_cluster0_load_rows) + assertTrue(before_cluster0_flush == after_cluster0_flush) + + assertTrue(before_cluster1_load_rows < after_cluster1_load_rows) + assertTrue(before_cluster1_flush < after_cluster1_flush) + + // case5 direct-to-BE stream load ignores compute_group and uses the receiving backend's compute group + before_cluster0_load_rows = get_be_metric(ipList[0], httpPortList[0], "load_rows"); + log.info("before_cluster0_load_rows : ${before_cluster0_load_rows}".toString()) + before_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total"); + log.info("before_cluster0_flush : ${before_cluster0_flush}".toString()) + + before_cluster1_load_rows = get_be_metric(ipList[1], httpPortList[1], "load_rows"); + log.info("before_cluster1_load_rows : ${before_cluster1_load_rows}".toString()) + before_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total"); + log.info("before_cluster1_flush : ${before_cluster1_flush}".toString()) + + streamLoad { + table "${tableName4}" + directToBe ipList[0], httpPortList[0].toInteger() + + set 'column_separator', ',' + set 'compute_group', 'stream_load_cluster_name1' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Direct-to-BE stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + } + } + sql "sync" + order_qt_direct_be_compute_group "SELECT count(*) FROM ${tableName4}" + + after_cluster0_load_rows = get_be_metric(ipList[0], httpPortList[0], "load_rows"); + log.info("after_cluster0_load_rows : ${after_cluster0_load_rows}".toString()) + after_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total"); + log.info("after_cluster0_flush : ${after_cluster0_flush}".toString()) + + after_cluster1_load_rows = get_be_metric(ipList[1], httpPortList[1], "load_rows"); + log.info("after_cluster1_load_rows : ${after_cluster1_load_rows}".toString()) + after_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total"); + log.info("after_cluster1_flush : ${after_cluster1_flush}".toString()) + + assertTrue(before_cluster0_load_rows < after_cluster0_load_rows) + assertTrue(before_cluster0_flush < after_cluster0_flush) + assertTrue(before_cluster1_load_rows == after_cluster1_load_rows) assertTrue(before_cluster1_flush == after_cluster1_flush) } finally { diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy index 81f2227ce44795..a6e7247f901aca 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy @@ -205,7 +205,7 @@ suite('test_rebalance_in_cloud', 'multi_cluster,docker') { sleep(1 * 1000) } GetDebugPoint().disableDebugPointForAllFEs("CloudTabletRebalancer.checkInflghtWarmUpCacheAsync.beNull"); - awaitUntil(10) { + awaitUntil(30) { def ret = sql_return_maparray """ADMIN SHOW REPLICA DISTRIBUTION FROM table100""" log.info("replica distribution table100: {}", ret) ret.any { row -> diff --git a/regression-test/suites/compaction/test_compaction_profile_action.groovy b/regression-test/suites/compaction/test_compaction_profile_action.groovy index 8882aab6795d1a..9eae22eac0d8dc 100644 --- a/regression-test/suites/compaction/test_compaction_profile_action.groovy +++ b/regression-test/suites/compaction/test_compaction_profile_action.groovy @@ -25,11 +25,6 @@ suite("test_compaction_profile_action", "p0") { def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort) - String backend_id = backendId_to_backendIP.keySet()[0] - def beHost = backendId_to_backendIP[backend_id] - def beHttpPort = backendId_to_backendHttpPort[backend_id] - def baseUrl = "http://${beHost}:${beHttpPort}/api/compaction/profile" - try { // Step 2: Create table, insert data, trigger compaction, wait for completion sql """ DROP TABLE IF EXISTS ${tableName} """ @@ -64,6 +59,10 @@ suite("test_compaction_profile_action", "p0") { assertTrue(tablets.size() > 0) def tablet = tablets[0] String tablet_id = tablet.TabletId + String backend_id = tablet.BackendId + def beHost = backendId_to_backendIP[backend_id] + def beHttpPort = backendId_to_backendHttpPort[backend_id] + def baseUrl = "http://${beHost}:${beHttpPort}/api/compaction/profile" // Trigger compaction and wait for completion trigger_and_wait_compaction(tableName, "cumulative") diff --git a/regression-test/suites/compaction/test_table_level_compaction_policy.groovy b/regression-test/suites/compaction/test_table_level_compaction_policy.groovy index 5edff4e38e0da5..0e1c70b02c9eea 100644 --- a/regression-test/suites/compaction/test_table_level_compaction_policy.groovy +++ b/regression-test/suites/compaction/test_table_level_compaction_policy.groovy @@ -219,22 +219,22 @@ suite("test_table_level_compaction_policy") { exception "only time series compaction policy support for time series config" } - test { - sql """ - CREATE TABLE ${tableName} ( - `c_custkey` int(11) NOT NULL COMMENT "", - `c_name` varchar(26) NOT NULL COMMENT "", - `c_address` varchar(41) NOT NULL COMMENT "", - `c_city` varchar(11) NOT NULL COMMENT "" - ) - DUPLICATE KEY (`c_custkey`) - DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1" - ); - """ - sql """sync""" + sql """ + CREATE TABLE ${tableName} ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + DUPLICATE KEY (`c_custkey`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """sync""" + test { sql """ alter table ${tableName} set ("compaction_policy" = "ok") """ diff --git a/regression-test/suites/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.groovy b/regression-test/suites/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.groovy new file mode 100644 index 00000000000000..9797453e901a8c --- /dev/null +++ b/regression-test/suites/datatype_p0/complex_types/test_light_schema_change_lazy_pruned_struct.groovy @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_light_schema_change_lazy_pruned_struct") { + sql "set batch_size = 4" + sql "set enable_prune_nested_column = true" + + sql "DROP TABLE IF EXISTS test_light_schema_change_lazy_pruned_struct" + sql """ + CREATE TABLE test_light_schema_change_lazy_pruned_struct ( + id INT, + s STRUCT, + b INT, + a INT + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ) + """ + + // Keep these rows in the old physical schema. After the ALTER, s.a is read by + // DefaultValueColumnIterator for this rowset. + sql """ + INSERT INTO test_light_schema_change_lazy_pruned_struct VALUES + (1, named_struct('b', 1, 'c', 'abc'), 21, 11), + (2, named_struct('b', 2, 'c', 'abc'), 22, 12), + (3, named_struct('b', 0, 'c', 'abc'), 23, 13), + (4, named_struct('b', 4, 'c', 'def'), 24, 14), + (5, named_struct('b', 5, 'c', 'abc'), 20, 15), + (6, named_struct('b', 6, 'c', 'abc'), 26, 10), + (7, named_struct('b', 7, 'c', 'abc'), 27, 17), + (8, named_struct('b', 8, 'c', 'def'), 28, 18) + """ + + sql """ + ALTER TABLE test_light_schema_change_lazy_pruned_struct + MODIFY COLUMN s STRUCT + """ + waitForSchemaChangeDone { + sql """ + SHOW ALTER TABLE COLUMN + WHERE TableName='test_light_schema_change_lazy_pruned_struct' + ORDER BY CreateTime DESC LIMIT 1 + """ + time 600 + } + + // Also keep a rowset written with the new physical schema in the same scan. + sql """ + INSERT INTO test_light_schema_change_lazy_pruned_struct VALUES + (9, named_struct('b', 9, 'c', 'abc', 'a', 90), 29, 19), + (10, named_struct('b', 0, 'c', 'abc', 'a', 100), 30, 20) + """ + + // s.a is a lazy output column, while s.b and s.c are predicate columns. + // On old rowsets s.a must replace its predicate-phase placeholders instead of + // appending defaults to them during lazy materialization. + order_qt_default_value_as_lazy_output """ + SELECT s.a + FROM test_light_schema_change_lazy_pruned_struct + WHERE s.b > 0 + AND s.c = 'abc' + AND b > 20 + AND a > 10 + """ + + // Exercise the inverse phase assignment: the default-backed child is used by + // a predicate and an old physical child is materialized lazily. + order_qt_default_value_as_predicate """ + SELECT s.b + FROM test_light_schema_change_lazy_pruned_struct + WHERE s.a IS NULL + AND s.c = 'abc' + AND b > 20 + AND a > 10 + """ +} diff --git a/regression-test/suites/datatype_p0/datetimev2/test_tz_load.groovy b/regression-test/suites/datatype_p0/datetimev2/test_tz_load.groovy index 49ec18b6541f3f..8a00dae736f47b 100644 --- a/regression-test/suites/datatype_p0/datetimev2/test_tz_load.groovy +++ b/regression-test/suites/datatype_p0/datetimev2/test_tz_load.groovy @@ -22,6 +22,9 @@ suite("test_tz_load", "nonConcurrent") { def s3Region = getS3Region() def ak = getS3AK() def sk = getS3SK() + def originGlobalTimeZone = sql("select @@global.time_zone")[0][0] + + try { sql "drop table if exists ${table1}" @@ -226,4 +229,8 @@ suite("test_tz_load", "nonConcurrent") { } sql "sync" qt_global_offset "select * from ${table1} order by id" -} \ No newline at end of file + } finally { + sql "SET GLOBAL time_zone = '${originGlobalTimeZone}'" + try_sql("drop table if exists ${table1}") + } +} diff --git a/regression-test/suites/demo_p0/test_flight_record.groovy b/regression-test/suites/demo_p0/test_flight_record.groovy index 652faf31305975..a69f820a2030e2 100644 --- a/regression-test/suites/demo_p0/test_flight_record.groovy +++ b/regression-test/suites/demo_p0/test_flight_record.groovy @@ -27,12 +27,29 @@ suite("test_flight_record", "nonConcurrent") { return } + // flightRecord starts the java flight recorder by the local jps/jcmd, so it only works when the + // frontend runs on the same machine as the regression runner, e.g. a local development cluster. + // Skip this demo when the frontend is not here, otherwise flightRecord will fail the suite + String feProcessName = "DorisFE" + boolean feOnThisMachine = false + try { + feOnThisMachine = "jps".execute().text.readLines().any { it.contains(feProcessName) } + } catch (Throwable t) { + // jps is shipped with the jdk, it doesn't exist if this machine only installs a jre + logger.info("Can not execute jps: ${t.getMessage()}") + } + if (!feOnThisMachine) { + logger.info("Process ${feProcessName} is not running on this machine, this case requires the " + + "frontend and the regression runner deployed together, skip test") + return + } + flightRecord { // whether delete jfr file after callback, default is true cleanUp true // the process name, default is DorisFE - processName "DorisFE" + processName feProcessName // the jcmd extra config, default is empty extraConfig(["jdk.ObjectAllocationSample#throttle=\"100 /ns\""]) diff --git a/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy b/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy index c17820fbb39112..64d3a1efb8d761 100644 --- a/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy +++ b/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy @@ -30,7 +30,7 @@ suite("oceanbase.md", "p2,external,oceanbase,external_docker,external_docker_oce sql """create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="root@test", - "password"="", + "password"="123456", "jdbc_url" = "jdbc:oceanbase://${externalEnvIp}:${oceanbase_port}/doris_test", "driver_url" = "${driver_url}", "driver_class" = "com.oceanbase.jdbc.Driver" diff --git a/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy b/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy index 6f9f1f7b0e0ca4..f20bfb60f38aa9 100644 --- a/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy +++ b/regression-test/suites/doc/sql-manual/sql-function/test_array_function.groovy @@ -520,4 +520,4 @@ suite("test_array_function_doc", "p0") { qt_sql """ SELECT ARRAY_SORTBY(x -> x[1], [[1,2],[3,4]]); """ qt_sql """ SELECT ARRAY_SORT([[1,2],[3,4]]); """ qt_sql """ SELECT ARRAY_REVERSE_SORT([[1,2],[3,4]]); """ -} \ No newline at end of file +} diff --git a/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy b/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy index ff4354f8dec142..08ae0da44c0463 100644 --- a/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy +++ b/regression-test/suites/external_table_p0/cache/test_file_cache_statistics.groovy @@ -45,6 +45,12 @@ suite("test_file_cache_statistics", "p0,external,nonConcurrent") { sql """set enable_file_cache=true""" sql """set disable_file_cache=false""" + // This case validates BlockFileCache counters. Keep upper-layer caches and scan scheduling + // from serving or cancelling the repeated read before it reaches BlockFileCache. + sql """set enable_sql_cache=false""" + sql """set enable_hive_sql_cache=false""" + sql """set enable_parquet_file_page_cache=false""" + sql """set parallel_pipeline_task_num=1""" // Check backend configuration prerequisites // Note: This test case assumes a single backend scenario. Testing with single backend is logically equivalent @@ -82,8 +88,10 @@ suite("test_file_cache_statistics", "p0,external,nonConcurrent") { sql """switch ${catalog_name}""" + // Pin the query to one partition file instead of racing all six file scan ranges under LIMIT 1. String querySql = """select * from ${catalog_name}.${ex_db_name}.parquet_partition_table - where l_orderkey=1 and l_partkey=1534 limit 1;""" + where nation='cn' and city='beijing' + and l_orderkey=1 and l_partkey=1534 limit 1;""" // load the table into file cache sql querySql diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy index ce0890fbccc1f6..2ef76dc8bd080d 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy @@ -223,6 +223,15 @@ suite("test_hive_topn_lazy_mat", "p0,external") { contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]") } + // Output aliases must not be used to resolve physical Hive column indices. + explain { + sql "select name as lazy_alias from orc_topn_lazy_mat_table order by id limit 10;" + contains("VMaterializeNode") + contains("column_descs_lists[[`name` text NULL]]") + contains("column_idxs_lists: [[1]]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__orc_topn_lazy_mat_table]") + } + explain { sql """ select a.name,length(a.name),a.value,b.*,a.* from parquet_topn_lazy_mat_table as a join orc_topn_lazy_mat_table as b on a.id = b.id order by a.name limit 10 """ diff --git a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy index 98f03c46e48959..981cd0e3b98676 100644 --- a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy +++ b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy @@ -177,25 +177,25 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("2", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("2", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("1", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("1", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -208,25 +208,25 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -239,25 +239,25 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -271,26 +271,26 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("8", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("8", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("7", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("7", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("6", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("6", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("9", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("9", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy index 93c8faf1ce522d..7d7672a4b3e6cc 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change_ddl.groovy @@ -193,7 +193,7 @@ suite("iceberg_schema_change_ddl", "p0,external") { sql """ ALTER TABLE ${table_name} MODIFY COLUMN non_col bigint""" exception "Column non_col does not exist" } - // not comment, the comment will be removed + // Omitted COMMENT preserves the existing comment. qt_before_no_comment "desc ${table_name}" sql """ ALTER TABLE ${table_name} MODIFY COLUMN col1 DOUBLE""" qt_after_no_comment "desc ${table_name}" @@ -305,7 +305,7 @@ suite("iceberg_schema_change_ddl", "p0,external") { // struct/complex type changes test { sql """ ALTER TABLE ${table_name} MODIFY COLUMN address STRING """ - exception "Cannot change column type" + exception "Modify column type from complex to primitive is not supported" } test { diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_arrow_flight_split_source.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_arrow_flight_split_source.groovy new file mode 100644 index 00000000000000..e721d0e9d8eab6 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_arrow_flight_split_source.groovy @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.sql.Connection +import java.sql.DriverManager + +import org.apache.doris.regression.util.JdbcUtils + +// Regression for https://github.com/apache/doris/issues/62259 +// +// Querying an Iceberg external table over Arrow Flight SQL in batch split mode used to fail +// (BE crash / "Split source X is released"). Arrow Flight executes a query in two phases: +// GetFlightInfo (plan + submit to BE) then DoGet (the client pulls results from the BE). In +// batch split mode the BE keeps scanning during DoGet and lazily fetches file splits from the +// FE via the fetchSplitBatch RPC, using an async SplitSource that the FE coordinator holds. The +// FE used to release that SplitSource at the end of GetFlightInfo, before the BE's DoGet, so the +// split fetch failed. The MySQL protocol is unaffected because plan + execute share one request. +// +// This test forces batch split mode on the Arrow Flight session and scans the table, which must +// now return all rows. +suite("test_iceberg_arrow_flight_split_source", "p0,external") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + // The bug only manifests over the Arrow Flight SQL protocol. Skip when it is not configured. + String arrowFlightHost = context.config.otherConfigs.get("extArrowFlightSqlHost") + if (arrowFlightHost == null || arrowFlightHost.isEmpty()) { + logger.info("extArrowFlightSqlHost is not configured, skip the test.") + return + } + + // The framework's arrow_flight_sql() helper always dials extArrowFlightSqlPort, but that + // configured value does not always match this cluster's real Arrow Flight SQL port (e.g. the + // external pipeline serves Arrow Flight on the default 8070 while extArrowFlightSqlPort is + // 8081). Read the live port from SHOW FRONTENDS and open our own connection against it, like + // the remote_doris tests do, so the test connects to this cluster's actual endpoint. + def frontends = sql """ show frontends """ + String arrowFlightPort = frontends[0][6].toString() + if (!arrowFlightPort.isInteger() || (arrowFlightPort as int) <= 0) { + logger.info("Arrow Flight SQL is disabled on this cluster (port=${arrowFlightPort}), skip the test.") + return + } + String arrowFlightUser = context.config.otherConfigs.get("extArrowFlightSqlUser") + String arrowFlightPassword = context.config.otherConfigs.get("extArrowFlightSqlPassword") + Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver") + String arrowFlightUrl = "jdbc:arrow-flight-sql://${arrowFlightHost}:${arrowFlightPort}" + + "/?useServerPrepStmts=false&useSSL=false&useEncryption=false" + + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalog_name = "test_iceberg_arrow_flight_split_source" + // sample_cow_orc has 1000 rows; with num_files_in_batch_mode=1 a plain scan uses batch mode. + String table = "${catalog_name}.format_v2.sample_cow_orc" + + sql """drop catalog if exists ${catalog_name}""" + sql """CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1" + );""" + + Connection flightConn = null + try { + // Baseline over the MySQL protocol (works regardless of the bug). + def expected = sql """ select count(*) from ${table}; """ + long expectedRows = (expected[0][0] as long) + assert expectedRows > 0 : "precondition: ${table} should not be empty" + + // A dedicated Arrow Flight SQL connection to this cluster's real port. Run a statement over + // it the same way arrow_flight_sql() does, via JdbcUtils.executeToList. + flightConn = DriverManager.getConnection(arrowFlightUrl, arrowFlightUser, arrowFlightPassword) + def flightSql = { String stmt -> + def (rows, meta) = JdbcUtils.executeToList(flightConn, stmt) + return rows + } + + // Force batch split mode on the Arrow Flight session (a separate session from the MySQL + // connection above, so the variables must be set here). With num_files_in_batch_mode=1 + // even a single-file scan builds the async SplitSource that triggers #62259. + flightSql """ set enable_external_table_batch_mode = true """ + flightSql """ set num_files_in_batch_mode = 1 """ + + // Make sure the Arrow Flight session really uses the batch SplitSource path, so the test + // cannot silently pass on the non-batch path. "approximate" only appears in batch mode. + def explainRows = flightSql """ explain select * from ${table} """ + boolean isBatch = explainRows.any { row -> + row.any { cell -> cell != null && cell.toString().contains("approximate") } + } + assert isBatch : "expected batch split mode (approximate) in the Arrow Flight plan, got: ${explainRows}" + + // The regression: a real data scan over Arrow Flight SQL (not count(*), which is pushed + // down and bypasses batch mode). Before the fix this failed with "Split source X is + // released" or crashed the BE; now it must return all rows. + def flightResult = flightSql """ select * from ${table} """ + assertEquals(expectedRows, (flightResult.size() as long)) + + // A second scan on the same connection also exercises cleanup of the previous query's + // deferred coordinator when the next query starts. + def flightLimited = flightSql """ select * from ${table} limit 10 """ + assert flightLimited.size() > 0 && flightLimited.size() <= 10 : "unexpected row count: ${flightLimited.size()}" + } finally { + // Close our own connection (best effort) so a dead endpoint cannot mask the real failure, + // then drop the catalog over the reliable MySQL connection. + if (flightConn != null) { + try { flightConn.close() } catch (Throwable ignore) {} + } + sql """drop catalog if exists ${catalog_name}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_jdbc_catalog.groovy index 11c887d25ceb0d..13f9f2a2c0c338 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_jdbc_catalog.groovy @@ -88,6 +88,14 @@ suite("test_iceberg_jdbc_catalog", "p0,external") { host_ips.add(f[1]) } host_ips = host_ips.unique() + Set localHostIps = ["127.0.0.1", "localhost", "::1"] as Set + java.util.Collections.list(java.net.NetworkInterface.getNetworkInterfaces()).each { networkInterface -> + java.util.Collections.list(networkInterface.getInetAddresses()).each { address -> + localHostIps.add(address.getHostAddress().split("%")[0]) + } + } + localHostIps.add(java.net.InetAddress.getLocalHost().getHostName()) + localHostIps.add(java.net.InetAddress.getLocalHost().getCanonicalHostName()) executeCommand("mkdir -p ${local_driver_dir}", false) if (!new File(local_driver_path).exists()) { @@ -97,9 +105,18 @@ suite("test_iceberg_jdbc_catalog", "p0,external") { executeCommand("/usr/bin/curl --max-time 600 ${mysql_driver_download_url} --output ${local_mysql_driver_path}", true) } for (def ip in host_ips) { - executeCommand("ssh -o StrictHostKeyChecking=no root@${ip} \"mkdir -p ${jdbc_drivers_dir}\"", false) - scpFiles("root", ip, local_driver_path, jdbc_drivers_dir, false) - scpFiles("root", ip, local_mysql_driver_path, jdbc_drivers_dir, false) + // Scenario: every FE/BE receives the JDBC drivers so distributed scans and FE failover + // do not depend on the regression runner sharing a filesystem with one cluster node. + if (localHostIps.contains(ip)) { + // A local test node must not require root SSH merely to install a JDBC driver. + executeCommand("mkdir -p ${jdbc_drivers_dir}", true) + executeCommand("cp -f ${local_driver_path} ${jdbc_drivers_dir}/${driver_name}", true) + executeCommand("cp -f ${local_mysql_driver_path} ${jdbc_drivers_dir}/${mysql_driver_name}", true) + } else { + executeCommand("ssh -o BatchMode=yes -o StrictHostKeyChecking=no root@${ip} \"mkdir -p ${jdbc_drivers_dir}\"", true) + scpFiles("root", ip, local_driver_path, jdbc_drivers_dir, false) + scpFiles("root", ip, local_mysql_driver_path, jdbc_drivers_dir, false) + } } try { @@ -215,6 +232,33 @@ suite("test_iceberg_jdbc_catalog", "p0,external") { assertTrue(desc.toString().contains("c_int")) assertTrue(desc.toString().contains("c_string")) + // Scenario TC09-JDBC: schema evolution and historical binding work through JDBC catalog. + String jdbcOldSnapshot = sql(""" + select snapshot_id + from test_datatypes\$snapshots + order by committed_at desc + limit 1 + """)[0][0].toString() + sql """alter table test_datatypes create tag jdbc_before_rename""" + sql """alter table test_datatypes rename column c_string jdbc_renamed_string""" + assertEquals([["hello"], ["world"], ["test"]], sql(""" + select c_string + from test_datatypes for version as of ${jdbcOldSnapshot} + order by c_int + """)) + assertEquals([["hello"], ["world"], ["test"]], sql(""" + select c_string + from test_datatypes@tag(jdbc_before_rename) + order by c_int + """)) + assertEquals([["hello"], ["world"], ["test"]], sql(""" + select jdbc_renamed_string from test_datatypes order by c_int + """)) + test { + sql """select c_string from test_datatypes""" + exception "Unknown column 'c_string'" + } + // Test: INSERT OVERWRITE sql """ INSERT OVERWRITE TABLE test_partitioned diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_mixed_file_formats.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_mixed_file_formats.groovy new file mode 100644 index 00000000000000..ebcb2affa29b79 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_mixed_file_formats.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_mixed_file_formats", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String catalogName = "test_iceberg_mixed_file_formats" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'iceberg', + 'iceberg.catalog.type' = 'rest', + 'uri' = 'http://${externalEnvIp}:${restPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.region' = 'us-east-1' + ) + """ + sql """switch ${catalogName}""" + sql """ set parallel_pipeline_task_num = 1; """ + order_qt_mixed_file_formats_files """ + SELECT lower(file_format), sum(record_count) + FROM test_db.mixed_file_format\$files + GROUP BY lower(file_format) + ORDER BY 1 + """ + + order_qt_mixed_file_formats_data """ + SELECT id, source + FROM test_db.mixed_file_format + ORDER BY id + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy new file mode 100644 index 00000000000000..012bc88e3d9faf --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_ddl.groovy @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_nested_schema_evolution_ddl", "p0,external,doris,external_docker,external_docker_doris") { + + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_nested_schema_evolution_ddl" + String dbName = "iceberg_nested_schema_evolution_db" + String tableName = "iceberg_nested_evolution" + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${restPort}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + );""" + + sql """switch ${catalogName};""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName};""" + + sql """set enable_fallback_to_original_planner=false;""" + + sql """drop table if exists ${tableName}""" + sql """ + CREATE TABLE ${tableName} ( + id INT NOT NULL, + s STRUCT, + arr ARRAY>, + m MAP>, + arr_scalar ARRAY, + m_scalar MAP + ); + """ + + sql """ + INSERT INTO ${tableName} VALUES ( + 1, + STRUCT(10, 'old'), + ARRAY(STRUCT(100)), + MAP('k', STRUCT(1000)), + ARRAY(7), + MAP('k', 70) + ) + """ + + sql """ALTER TABLE ${tableName} ADD COLUMN s.c STRING NULL COMMENT 'new nested field'""" + sql """ALTER TABLE ${tableName} ADD COLUMN s.first_pos STRING NULL FIRST""" + sql """ALTER TABLE ${tableName} ADD COLUMN s.after_a STRING NULL AFTER a""" + sql """ALTER TABLE ${tableName} ADD COLUMN arr.element.y INT NULL""" + sql """ALTER TABLE ${tableName} ADD COLUMN arr.element.after_x INT NULL AFTER x""" + sql """ALTER TABLE ${tableName} ADD COLUMN m.value.y INT NULL""" + sql """ALTER TABLE ${tableName} ADD COLUMN m.value.after_v INT NULL AFTER v""" + sql """ALTER TABLE ${tableName} ADD COLUMN s.drop_me STRING NULL""" + sql """ALTER TABLE ${tableName} ADD COLUMN arr.element.drop_me INT NULL""" + sql """ALTER TABLE ${tableName} ADD COLUMN m.value.drop_me INT NULL""" + + test { + sql """ALTER TABLE ${tableName} ADD COLUMN s.required_field INT NOT NULL""" + exception "New nested field 's.required_field' must be nullable" + } + + sql """ALTER TABLE ${tableName} MODIFY COLUMN s.a BIGINT""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr.element.x BIGINT""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN m.value.v BIGINT""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr_scalar.element BIGINT""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value BIGINT""" + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.`key` BIGINT""" + exception "Cannot modify MAP key nested column" + } + + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr_scalar.element COMMENT 'array element comment'""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value COMMENT 'map value comment'""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr_scalar.element BIGINT COMMENT 'array element comment'""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value BIGINT COMMENT 'map value comment'""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr_scalar.element BIGINT COMMENT ''""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.value BIGINT COMMENT ''""" + exception "Iceberg does not support comments on collection element or value fields" + } + test { + sql """ALTER TABLE ${tableName} MODIFY COLUMN m_scalar.`key` COMMENT 'map key comment'""" + exception "Cannot modify comment MAP key nested column" + } + + sql """ALTER TABLE ${tableName} RENAME COLUMN s.c TO c2""" + sql """ALTER TABLE ${tableName} RENAME COLUMN arr.element.y TO y2""" + sql """ALTER TABLE ${tableName} RENAME COLUMN m.value.y TO y2""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN s.`c2` COMMENT 'renamed struct field'""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN arr.element.y2 COMMENT 'renamed array element field'""" + sql """ALTER TABLE ${tableName} MODIFY COLUMN m.value.y2 COMMENT 'renamed map value field'""" + sql """ALTER TABLE ${tableName} DROP COLUMN s.drop_me""" + sql """ALTER TABLE ${tableName} DROP COLUMN arr.element.drop_me""" + sql """ALTER TABLE ${tableName} DROP COLUMN m.value.drop_me""" + + sql """ + INSERT INTO ${tableName} VALUES ( + 2, + STRUCT('first', 20, 'after_a', 'new', 'c2'), + ARRAY(STRUCT(200, 202, 201)), + MAP('k', STRUCT(2000, 2002, 2001)), + ARRAY(8), + MAP('k', 80) + ) + """ + + qt_desc """ + SELECT COLUMN_NAME, COLUMN_TYPE + FROM ${catalogName}.information_schema.columns + WHERE TABLE_SCHEMA = '${dbName}' AND TABLE_NAME = '${tableName}' + ORDER BY ORDINAL_POSITION + """ + + order_qt_query_rows """ + SELECT id, + element_at(s, 'first_pos'), + element_at(s, 'a'), + element_at(s, 'after_a'), + element_at(s, 'c2'), + element_at(arr[1], 'x'), + element_at(arr[1], 'after_x'), + element_at(arr[1], 'y2'), + element_at(m['k'], 'v'), + element_at(m['k'], 'after_v'), + element_at(m['k'], 'y2'), + arr_scalar[1], + m_scalar['k'] + FROM ${tableName} + ORDER BY id + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy new file mode 100644 index 00000000000000..96725c2aedd884 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_nested_schema_evolution_spark_doris_interop.groovy @@ -0,0 +1,553 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonSlurper + +suite("test_iceberg_nested_schema_evolution_spark_doris_interop", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String catalogName = "test_iceberg_nested_schema_evolution_spark_doris_interop" + String dbName = "iceberg_nested_schema_evolution_interop_db" + String dorisTable = "doris_nested_evolution_to_spark" + String commentTable = "doris_nested_comment_semantics" + String sparkTable = "spark_nested_evolution_to_doris" + String mixedCaseTable = "spark_mixed_case_nested_collision" + String requiredTable = "spark_required_nested_evolution" + String advancedTable = "spark_deep_nested_evolution" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${restPort}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0" + ); + """ + + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner=false""" + + spark_iceberg """CREATE DATABASE IF NOT EXISTS demo.${dbName}""" + + sql """ + CREATE TABLE ${commentTable} ( + id INT, + info STRUCT< + metric:INT COMMENT 'metric doc', + note:STRING COMMENT 'old note', + clear_me:STRING COMMENT 'clear doc', + payload:STRUCT< + name:STRING COMMENT 'name doc', + count:INT COMMENT 'count doc' + > COMMENT 'payload doc' + > + ) + """ + sql """ALTER TABLE ${commentTable} MODIFY COLUMN info.note COMMENT 'new note'""" + sql """ALTER TABLE ${commentTable} MODIFY COLUMN info.metric BIGINT""" + sql """ALTER TABLE ${commentTable} MODIFY COLUMN info.clear_me STRING COMMENT ''""" + sql """ + ALTER TABLE ${commentTable} MODIFY COLUMN info.payload + STRUCT + """ + + String loadTableUrl = "http://${externalEnvIp}:${restPort}/v1/namespaces/${dbName}/tables/${commentTable}" + def loadTableResponse = new JsonSlurper().parseText(new URL(loadTableUrl).getText("UTF-8")) + def tableMetadata = loadTableResponse.metadata + def currentSchema = tableMetadata.schemas.find { + it["schema-id"] == tableMetadata["current-schema-id"] + } + assertNotNull(currentSchema, "current schema should exist in Iceberg metadata") + def infoColumn = currentSchema.fields.find { it.name == "info" } + assertNotNull(infoColumn, "info column should exist in Iceberg metadata") + def infoFields = infoColumn.type.fields.collectEntries { [(it.name): it] } + assertEquals("long", infoFields.metric.type) + assertEquals("metric doc", infoFields.metric.doc) + assertEquals("new note", infoFields.note.doc) + assertTrue(infoFields.clear_me.doc == null || infoFields.clear_me.doc == "") + assertEquals("payload doc", infoFields.payload.doc) + def payloadFields = infoFields.payload.type.fields.collectEntries { [(it.name): it] } + assertTrue(payloadFields.name.doc == null || payloadFields.name.doc == "") + assertEquals("long", payloadFields.count.type) + assertEquals("count doc", payloadFields.count.doc) + + sql """ + CREATE TABLE ${dorisTable} ( + id INT NOT NULL, + info STRUCT, + events ARRAY>, + attrs MAP> + ) + """ + sql """ + INSERT INTO ${dorisTable} VALUES ( + 1, + STRUCT(10, 'doris_before'), + ARRAY(STRUCT(100)), + MAP('k', STRUCT(1000)) + ) + """ + + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_added STRING NULL COMMENT 'added by doris'""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_first STRING NULL FIRST""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.doris_after_metric STRING NULL AFTER metric""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_score INT NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_first_score INT NULL FIRST""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.doris_after_score INT NULL AFTER score""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_code INT NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_first_code INT NULL FIRST""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.doris_after_code INT NULL AFTER code""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN info.drop_me STRING NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN events.element.drop_me INT NULL""" + sql """ALTER TABLE ${dorisTable} ADD COLUMN attrs.value.drop_me INT NULL""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN info.metric BIGINT""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN events.element.score BIGINT""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN attrs.value.code BIGINT""" + sql """ALTER TABLE ${dorisTable} RENAME COLUMN info.doris_added TO doris_renamed""" + sql """ALTER TABLE ${dorisTable} RENAME COLUMN events.element.doris_score TO doris_score_renamed""" + sql """ALTER TABLE ${dorisTable} RENAME COLUMN attrs.value.doris_code TO doris_code_renamed""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN info.doris_renamed COMMENT 'renamed by doris'""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN events.element.doris_score_renamed COMMENT 'renamed by doris'""" + sql """ALTER TABLE ${dorisTable} MODIFY COLUMN attrs.value.doris_code_renamed COMMENT 'renamed by doris'""" + sql """ALTER TABLE ${dorisTable} DROP COLUMN info.drop_me""" + sql """ALTER TABLE ${dorisTable} DROP COLUMN events.element.drop_me""" + sql """ALTER TABLE ${dorisTable} DROP COLUMN attrs.value.drop_me""" + + spark_iceberg """REFRESH TABLE demo.${dbName}.${dorisTable}""" + spark_iceberg """ + INSERT INTO demo.${dbName}.${dorisTable} VALUES ( + 2, + NAMED_STRUCT('doris_first', 'spark_first', + 'metric', CAST(20 AS BIGINT), + 'doris_after_metric', 'spark_after_metric', + 'label', 'spark_after_doris', + 'doris_renamed', 'spark_can_write_doris_field'), + ARRAY(NAMED_STRUCT('doris_first_score', 202, + 'score', CAST(200 AS BIGINT), + 'doris_after_score', 203, + 'doris_score_renamed', 201)), + MAP('k', NAMED_STRUCT('doris_first_code', 2002, + 'code', CAST(2000 AS BIGINT), + 'doris_after_code', 2003, + 'doris_code_renamed', 2001)) + ) + """ + + sql """refresh table ${dbName}.${dorisTable}""" + def dorisDrivenSparkRows = spark_iceberg """ + SELECT id, + info.doris_first, + info.metric, + info.doris_after_metric, + info.label, + info.doris_renamed, + events[0].doris_first_score, + events[0].score, + events[0].doris_after_score, + events[0].doris_score_renamed, + attrs['k'].doris_first_code, + attrs['k'].code, + attrs['k'].doris_after_code, + attrs['k'].doris_code_renamed + FROM demo.${dbName}.${dorisTable} + ORDER BY id + """ + String dorisDrivenDorisQuery = """ + SELECT id, + element_at(info, 'doris_first'), + element_at(info, 'metric'), + element_at(info, 'doris_after_metric'), + element_at(info, 'label'), + element_at(info, 'doris_renamed'), + element_at(events[1], 'doris_first_score'), + element_at(events[1], 'score'), + element_at(events[1], 'doris_after_score'), + element_at(events[1], 'doris_score_renamed'), + element_at(attrs['k'], 'doris_first_code'), + element_at(attrs['k'], 'code'), + element_at(attrs['k'], 'doris_after_code'), + element_at(attrs['k'], 'doris_code_renamed') + FROM ${dorisTable} + ORDER BY id + """ + order_qt_doris_driven_rows dorisDrivenDorisQuery + def dorisDrivenDorisRows = sql dorisDrivenDorisQuery + assertSparkDorisResultEquals(dorisDrivenSparkRows, dorisDrivenDorisRows) + + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.${sparkTable}; + DROP TABLE IF EXISTS demo.${dbName}.${mixedCaseTable}; + CREATE TABLE demo.${dbName}.${mixedCaseTable} ( + Id INT, + Label STRING, + Info STRUCT + ) USING iceberg; + CREATE TABLE demo.${dbName}.${sparkTable} ( + id INT, + info STRUCT, + events ARRAY>, + attrs MAP> + ) USING iceberg; + INSERT INTO demo.${dbName}.${sparkTable} VALUES ( + 1, + NAMED_STRUCT('metric', 10, 'label', 'spark_before'), + ARRAY(NAMED_STRUCT('score', 100)), + MAP('k', NAMED_STRUCT('code', 1000)) + ); + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_added STRING; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_first STRING FIRST; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.spark_after_metric STRING AFTER metric; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_score INT; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_first_score INT FIRST; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.spark_after_score INT AFTER score; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_code INT; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_first_code INT FIRST; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.spark_after_code INT AFTER code; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN info.drop_me STRING; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN events.element.drop_me INT; + ALTER TABLE demo.${dbName}.${sparkTable} ADD COLUMN attrs.value.drop_me INT; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN info.metric TYPE BIGINT; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN events.element.score TYPE BIGINT; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN attrs.value.code TYPE BIGINT; + ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN info.spark_added TO spark_renamed; + ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN events.element.spark_score TO spark_score_renamed; + ALTER TABLE demo.${dbName}.${sparkTable} RENAME COLUMN attrs.value.spark_code TO spark_code_renamed; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN info.spark_renamed COMMENT 'renamed by spark'; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN events.element.spark_score_renamed COMMENT 'renamed by spark'; + ALTER TABLE demo.${dbName}.${sparkTable} ALTER COLUMN attrs.value.spark_code_renamed COMMENT 'renamed by spark'; + ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN info.drop_me; + ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN events.element.drop_me; + ALTER TABLE demo.${dbName}.${sparkTable} DROP COLUMN attrs.value.drop_me; + INSERT INTO demo.${dbName}.${sparkTable} VALUES ( + 2, + NAMED_STRUCT('spark_first', 'spark_first_field', + 'metric', CAST(20 AS BIGINT), + 'spark_after_metric', 'spark_after_metric_field', + 'label', 'spark_after', + 'spark_renamed', 'spark_new_field'), + ARRAY(NAMED_STRUCT('spark_first_score', 202, + 'score', CAST(200 AS BIGINT), + 'spark_after_score', 203, + 'spark_score_renamed', 201)), + MAP('k', NAMED_STRUCT('spark_first_code', 2002, + 'code', CAST(2000 AS BIGINT), + 'spark_after_code', 2003, + 'spark_code_renamed', 2001)) + ); + """ + + sql """refresh catalog ${catalogName}""" + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + test { + sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN id STRING NULL""" + exception "conflicts with existing Iceberg field 'Id' (case-insensitive)" + } + test { + sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN (id STRING)""" + exception "conflicts with existing Iceberg field 'Id' (case-insensitive)" + } + test { + sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN (new_field STRING, NEW_FIELD STRING)""" + exception "conflicts with another requested column (case-insensitive)" + } + test { + sql """ALTER TABLE ${mixedCaseTable} RENAME COLUMN label TO id""" + exception "conflicts with existing Iceberg field 'Id' (case-insensitive)" + } + sql """ALTER TABLE ${mixedCaseTable} RENAME COLUMN label TO label""" + + test { + sql """ALTER TABLE ${mixedCaseTable} ADD COLUMN info.metric STRING NULL""" + exception "conflicts with existing Iceberg field 'Info.Metric' (case-insensitive)" + } + test { + sql """ALTER TABLE ${mixedCaseTable} RENAME COLUMN info.label TO metric""" + exception "conflicts with existing Iceberg field 'Info.Metric' (case-insensitive)" + } + + qt_spark_driven_schema """ + SELECT COLUMN_NAME, COLUMN_TYPE + FROM ${catalogName}.information_schema.columns + WHERE TABLE_SCHEMA = '${dbName}' AND TABLE_NAME = '${sparkTable}' + ORDER BY ORDINAL_POSITION + """ + + String sparkDrivenDorisQuery = """ + SELECT id, + element_at(info, 'spark_first'), + element_at(info, 'metric'), + element_at(info, 'spark_after_metric'), + element_at(info, 'label'), + element_at(info, 'spark_renamed'), + element_at(events[1], 'spark_first_score'), + element_at(events[1], 'score'), + element_at(events[1], 'spark_after_score'), + element_at(events[1], 'spark_score_renamed'), + element_at(attrs['k'], 'spark_first_code'), + element_at(attrs['k'], 'code'), + element_at(attrs['k'], 'spark_after_code'), + element_at(attrs['k'], 'spark_code_renamed') + FROM ${sparkTable} + ORDER BY id + """ + order_qt_spark_driven_rows_before_write sparkDrivenDorisQuery + + sql """ + INSERT INTO ${sparkTable} VALUES ( + 3, + STRUCT('doris_first_field', 30, 'doris_after_metric_field', + 'doris_after_spark', 'doris_can_write_spark_field'), + ARRAY(STRUCT(302, 300, 303, 301)), + MAP('k', STRUCT(3002, 3000, 3003, 3001)) + ) + """ + + spark_iceberg """REFRESH TABLE demo.${dbName}.${sparkTable}""" + def sparkDrivenSparkRows = spark_iceberg """ + SELECT id, + info.spark_first, + info.metric, + info.spark_after_metric, + info.label, + info.spark_renamed, + events[0].spark_first_score, + events[0].score, + events[0].spark_after_score, + events[0].spark_score_renamed, + attrs['k'].spark_first_code, + attrs['k'].code, + attrs['k'].spark_after_code, + attrs['k'].spark_code_renamed + FROM demo.${dbName}.${sparkTable} + ORDER BY id + """ + order_qt_spark_driven_rows_after_write sparkDrivenDorisQuery + def sparkDrivenDorisRows = sql sparkDrivenDorisQuery + assertSparkDorisResultEquals(sparkDrivenSparkRows, sparkDrivenDorisRows) + + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.${requiredTable}; + CREATE TABLE demo.${dbName}.${requiredTable} ( + id INT, + info STRUCT + ) USING iceberg; + INSERT INTO demo.${dbName}.${requiredTable} VALUES ( + 1, + NAMED_STRUCT('required_metric', 10, 'required_label', 'old-label') + ); + """ + + sql """refresh catalog ${catalogName}""" + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + sql """ALTER TABLE ${requiredTable} MODIFY COLUMN info.required_metric BIGINT""" + sql """ALTER TABLE ${requiredTable} MODIFY COLUMN info.required_label STRING NULL""" + test { + sql """ALTER TABLE ${requiredTable} MODIFY COLUMN info.required_label STRING NOT NULL""" + exception "Can not change nullable column info.required_label to not null" + } + + String requiredTableUrl = + "http://${externalEnvIp}:${restPort}/v1/namespaces/${dbName}/tables/${requiredTable}" + def requiredTableMetadata = new JsonSlurper().parseText( + new URL(requiredTableUrl).getText("UTF-8")).metadata + def requiredSchema = requiredTableMetadata.schemas.find { + it["schema-id"] == requiredTableMetadata["current-schema-id"] + } + def requiredInfo = requiredSchema.fields.find { it.name == "info" } + def requiredInfoFields = requiredInfo.type.fields.collectEntries { [(it.name): it] } + assertEquals("long", requiredInfoFields.required_metric.type) + assertTrue(requiredInfoFields.required_metric.required) + assertFalse(requiredInfoFields.required_label.required) + + sql """ + INSERT INTO ${requiredTable} VALUES ( + 2, + STRUCT(20, NULL) + ) + """ + + String requiredDorisQuery = """ + SELECT id, + element_at(info, 'required_metric'), + element_at(info, 'required_label') + FROM ${requiredTable} + ORDER BY id + """ + order_qt_required_nested_rows requiredDorisQuery + spark_iceberg """REFRESH TABLE demo.${dbName}.${requiredTable}""" + def requiredSparkRows = spark_iceberg """ + SELECT id, info.required_metric, info.required_label + FROM demo.${dbName}.${requiredTable} + ORDER BY id + """ + def requiredDorisRows = sql requiredDorisQuery + assertSparkDorisResultEquals(requiredSparkRows, requiredDorisRows) + + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.${advancedTable}; + CREATE TABLE demo.${dbName}.${advancedTable} ( + id INT, + root STRUCT< + first_field: STRING, + middle_field: STRING, + last_field: STRING, + items: ARRAY>>> + > + ) USING iceberg; + INSERT INTO demo.${dbName}.${advancedTable} VALUES ( + 1, + NAMED_STRUCT( + 'first_field', 'first-old', + 'middle_field', 'middle-old', + 'last_field', 'last-old', + 'items', ARRAY(NAMED_STRUCT( + 'attrs', MAP('k', NAMED_STRUCT( + 'score', CAST(1.5 AS FLOAT), + 'amount', CAST(12.34 AS DECIMAL(9, 2)), + 'note', 'old-note' + )) + )) + ) + ); + """ + + sql """refresh catalog ${catalogName}""" + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + sql """ALTER TABLE ${advancedTable} MODIFY COLUMN root.last_field STRING FIRST""" + sql """ALTER TABLE ${advancedTable} MODIFY COLUMN root.first_field STRING AFTER middle_field""" + sql """ALTER TABLE ${advancedTable} MODIFY COLUMN root.items.element.attrs.value.score DOUBLE""" + sql """ALTER TABLE ${advancedTable} MODIFY COLUMN root.items.element.attrs.value.amount DECIMAL(18, 2)""" + test { + sql """ALTER TABLE ${advancedTable} + MODIFY COLUMN root.items.element.attrs.value.amount DECIMAL(18, 3)""" + exception "Cannot change column type" + } + test { + sql """ALTER TABLE ${advancedTable} + MODIFY COLUMN root.items.element.attrs.value.score FLOAT""" + exception "Cannot change column type" + } + sql """ALTER TABLE ${advancedTable} + ADD COLUMN root.items.element.attrs.value.added STRING NULL AFTER amount""" + sql """ALTER TABLE ${advancedTable} + RENAME COLUMN root.items.element.attrs.value.added TO renamed""" + sql """ALTER TABLE ${advancedTable} + MODIFY COLUMN root.items.element.attrs.value.renamed COMMENT 'deep field'""" + sql """ALTER TABLE ${advancedTable} + MODIFY COLUMN root.items.element.attrs.value.note STRING FIRST""" + + String advancedTableUrl = + "http://${externalEnvIp}:${restPort}/v1/namespaces/${dbName}/tables/${advancedTable}" + def advancedTableMetadata = new JsonSlurper().parseText( + new URL(advancedTableUrl).getText("UTF-8")).metadata + def advancedSchema = advancedTableMetadata.schemas.find { + it["schema-id"] == advancedTableMetadata["current-schema-id"] + } + def rootField = advancedSchema.fields.find { it.name == "root" } + assertEquals(["last_field", "middle_field", "first_field", "items"], + rootField.type.fields.collect { it.name }) + def itemsField = rootField.type.fields.find { it.name == "items" } + def attrsField = itemsField.type.element.fields.find { it.name == "attrs" } + def deepFields = attrsField.type.value.fields + assertEquals(["note", "score", "amount", "renamed"], deepFields.collect { it.name }) + assertEquals("double", deepFields.find { it.name == "score" }.type) + assertEquals("decimal(18, 2)", deepFields.find { it.name == "amount" }.type) + assertEquals("deep field", deepFields.find { it.name == "renamed" }.doc) + + sql """ALTER TABLE ${advancedTable} + DROP COLUMN root.items.element.attrs.value.renamed""" + + def droppedFieldMetadata = new JsonSlurper().parseText( + new URL(advancedTableUrl).getText("UTF-8")).metadata + def droppedFieldSchema = droppedFieldMetadata.schemas.find { + it["schema-id"] == droppedFieldMetadata["current-schema-id"] + } + def droppedRootField = droppedFieldSchema.fields.find { it.name == "root" } + def droppedItemsField = droppedRootField.type.fields.find { it.name == "items" } + def droppedAttrsField = droppedItemsField.type.element.fields.find { it.name == "attrs" } + assertEquals(["note", "score", "amount"], + droppedAttrsField.type.value.fields.collect { it.name }) + + sql """ + INSERT INTO ${advancedTable} VALUES ( + 2, + STRUCT( + 'last-new', + 'middle-new', + 'first-new', + ARRAY(STRUCT(MAP('k', STRUCT( + 'new-note', + CAST(2.5 AS DOUBLE), + CAST(56.78 AS DECIMAL(18, 2)) + )))) + ) + ) + """ + + String advancedDorisQuery = """ + SELECT id, + element_at(root, 'last_field'), + element_at(root, 'middle_field'), + element_at(root, 'first_field'), + element_at(element_at(element_at(root, 'items')[1], 'attrs')['k'], 'note'), + element_at(element_at(element_at(root, 'items')[1], 'attrs')['k'], 'score'), + element_at(element_at(element_at(root, 'items')[1], 'attrs')['k'], 'amount') + FROM ${advancedTable} + ORDER BY id + """ + order_qt_deep_nested_rows advancedDorisQuery + spark_iceberg """REFRESH TABLE demo.${dbName}.${advancedTable}""" + def advancedSparkRows = spark_iceberg """ + SELECT id, + root.last_field, + root.middle_field, + root.first_field, + root.items[0].attrs['k'].note, + root.items[0].attrs['k'].score, + root.items[0].attrs['k'].amount + FROM demo.${dbName}.${advancedTable} + ORDER BY id + """ + def advancedDorisRows = sql advancedDorisQuery + assertSparkDorisResultEquals(advancedSparkRows, advancedDorisRows) +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.groovy new file mode 100644 index 00000000000000..a9fcfb4aeb7fa7 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_dual_relation_matrix.groovy @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_schema_dual_relation_matrix", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_schema_dual_relation_matrix" + String dbName = "iceberg_schema_dual_relation_db" + String tableName = "dual_schema_timeline" + + def latestSnapshotId = { + List> rows = spark_iceberg """ + select snapshot_id + from demo.${dbName}.${tableName}.snapshots + order by committed_at desc + limit 1 + """ + assertEquals(1, rows.size()) + return rows[0][0].toString() + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri'='http://${externalEnvIp}:${restPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.region'='us-east-1' + ) + """ + + try { + spark_iceberg_multi """ + create database if not exists demo.${dbName}; + drop table if exists demo.${dbName}.${tableName}; + create table demo.${dbName}.${tableName} ( + id int, + old_name string, + info struct + ) using iceberg + tblproperties ('format-version'='2', 'write.format.default'='parquet'); + insert into demo.${dbName}.${tableName} + values (1, 'old-1', named_struct('added', 10, 'keep', 11)); + """ + String oldSnapshot = latestSnapshotId() + + spark_iceberg_multi """ + alter table demo.${dbName}.${tableName} rename column old_name to new_name; + alter table demo.${dbName}.${tableName} + rename column info.added to renamed; + insert into demo.${dbName}.${tableName} + values (2, 'new-2', named_struct('renamed', 20, 'keep', 21)); + """ + String newSnapshot = latestSnapshotId() + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + // Scenario TC07-baseline: each historical relation resolves its own schema in isolation. + assertEquals([[1, "old-1", 10]], sql(""" + select id, old_name, info.added + from ${tableName} for version as of ${oldSnapshot} + order by id + """)) + assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + select id, new_name, info.renamed + from ${tableName} for version as of ${newSnapshot} + order by id + """)) + + // Scenario TC07-join negative contract: + // two historical relations in one statement currently reuse the first schema. + test { + sql """ + select o.id, o.old_name, n.new_name + from ( + select id, old_name + from ${tableName} for version as of ${oldSnapshot} + ) o + join ( + select id, new_name + from ${tableName} for version as of ${newSnapshot} + ) n on o.id = n.id + order by o.id + """ + exception "Unknown column 'new_name'" + } + + // Scenario TC07-reverse-join: binding must be independent of relation order. + test { + sql """ + select n.id, n.new_name, o.old_name + from ( + select id, new_name + from ${tableName} for version as of ${newSnapshot} + ) n + join ( + select id, old_name + from ${tableName} for version as of ${oldSnapshot} + ) o on n.id = o.id + order by n.id + """ + exception "Unknown column 'old_name'" + } + + // Scenario TC07-union: top-level historical schemas stay relation-local. + test { + sql """ + select id, old_name as name_value + from ${tableName} for version as of ${oldSnapshot} + union all + select id, new_name as name_value + from ${tableName} for version as of ${newSnapshot} + order by id, name_value + """ + exception "Unknown column 'new_name'" + } + + // Scenario TC07-nested-union: nested field lookup is also relation-local. + test { + sql """ + select id, info.added as nested_value + from ${tableName} for version as of ${oldSnapshot} + union all + select id, info.renamed as nested_value + from ${tableName} for version as of ${newSnapshot} + order by id, nested_value + """ + exception "No such struct field 'renamed'" + } + + // Scenario TC07-CTE: CTE boundaries must not collapse snapshot schemas. + test { + sql """ + with old_ref as ( + select id, old_name + from ${tableName} for version as of ${oldSnapshot} + ), new_ref as ( + select id, new_name + from ${tableName} for version as of ${newSnapshot} + ) + select old_ref.id, old_ref.old_name, new_ref.new_name + from old_ref join new_ref on old_ref.id = new_ref.id + order by old_ref.id + """ + exception "Unknown column 'new_name'" + } + + // Scenario TC07-correlated-subquery: subqueries require an independent schema. + test { + sql """ + select o.id, o.old_name + from ${tableName} for version as of ${oldSnapshot} o + where exists ( + select 1 + from ${tableName} for version as of ${newSnapshot} n + where n.id = o.id and n.new_name is not null + ) + order by o.id + """ + exception "Unknown column 'new_name'" + } + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_equality_delete_time_travel.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_equality_delete_time_travel.groovy new file mode 100644 index 00000000000000..5e9ab170b5bb6f --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_equality_delete_time_travel.groovy @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_schema_equality_delete_time_travel", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_schema_equality_delete_time_travel" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri'='http://${externalEnvIp}:${restPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.region'='us-east-1' + ) + """ + sql """switch ${catalogName}""" + sql """use multi_catalog""" + + try { + ["par", "orc"].each { format -> + String tableName = "equality_delete_${format}_1" + List snapshotIds = sql(""" + select snapshot_id + from ${tableName}\$snapshots + order by committed_at, snapshot_id + """).collect { row -> row[0].toString() } + assertTrue(snapshotIds.size() >= 7, + "Equality-delete fixture must retain the schema timeline for ${tableName}") + + String oldSnapshot = snapshotIds[0] + String renamedSnapshot = snapshotIds[2] + String latestSnapshot = snapshotIds.last() + + // Scenario TC04-EQ-S04: equality deletes before rename stay bound to the old field ID. + assertEquals([ + [1, "smith", "a"], + [2, "danny", "b"], + [3, "alice", "c"], + [4, "bob", "d"] + ], sql(""" + select id, name, data + from ${tableName} for version as of ${oldSnapshot} + order by id + """)) + test { + sql """ + select new_new_id + from ${tableName} for version as of ${oldSnapshot} + """ + exception "Unknown column 'new_new_id'" + } + + // Scenario TC04-EQ-S04/S08: the renamed-key snapshot applies equality deletes by field ID. + List> renamedRows = sql(""" + select new_id, name, data + from ${tableName} for version as of ${renamedSnapshot} + order by new_id, name, data + """) + String lastName = format == "par" ? "parker" : "orcker" + assertEquals([ + [1, "smith2", "aa"], + [2, "danny2", "bb"], + [3, "alice", "c"], + [4, "bob", "e"], + [5, "dennis", "f"], + [6, "jasson", "g"], + [7, lastName, "h"] + ], renamedRows) + + // Scenario TC04-EQ-S07: re-added id has a new field ID and must not match old delete keys. + List> latestRowsV2 + sql """set enable_file_scanner_v2=true""" + latestRowsV2 = sql(""" + select new_new_id, new_name, data, id + from ${tableName} for version as of ${latestSnapshot} + order by new_new_id, new_name, data, id + """) + assertEquals([ + [1, "smith4", "aaaa", 1], + [2, "danny2", "bb", null], + [3, "alice", "c", null], + [4, "bob3", "eee", null], + [5, "dennis2", "ff", null], + [6, "jasson", "g", null], + [7, lastName, "h", null] + ], latestRowsV2) + + // Scenario TC04-EQ-R13: legacy and V2 scanners apply the same equality deletes. + sql """set enable_file_scanner_v2=false""" + List> latestRowsLegacy = sql(""" + select new_new_id, new_name, data, id + from ${tableName} for version as of ${latestSnapshot} + order by new_new_id, new_name, data, id + """) + assertEquals(latestRowsV2, latestRowsLegacy) + + // Scenario TC04-EQ-query-shapes: projection, predicate and aggregation share delete semantics. + assertEquals(latestRowsLegacy.size().toLong(), sql(""" + select count(*) from ${tableName} for version as of ${latestSnapshot} + """)[0][0]) + assertEquals(latestRowsLegacy.findAll { row -> row[0].toString().toInteger() >= 4 }.size().toLong(), + sql(""" + select count(*) + from ${tableName} for version as of ${latestSnapshot} + where new_new_id >= 4 + """)[0][0]) + + // Old refs remain readable after all equality-delete/schema commits. + assertEquals(4L, sql(""" + select count(*) + from ${tableName} for version as of ${oldSnapshot} + """)[0][0]) + } + } finally { + sql """set enable_file_scanner_v2=true""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_metadata_atomicity_matrix.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_metadata_atomicity_matrix.groovy new file mode 100644 index 00000000000000..45307819e599a2 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_metadata_atomicity_matrix.groovy @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_schema_metadata_atomicity_matrix", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_schema_metadata_atomicity_matrix" + String dbName = "iceberg_schema_metadata_atomicity_db" + String tableName = "metadata_timeline" + + def snapshotCount = { + return sql("""select count(*) from ${tableName}\$snapshots""")[0][0].toString().toInteger() + } + def schemaCount = { + return sql("""select count(*) from ${tableName}\$metadata_log_entries""")[0][0] + .toString().toInteger() + } + def allDescText = { + return sql("""desc ${tableName}""").flatten().collect { + it == null ? "" : it.toString() + }.join(" ") + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri'='http://${externalEnvIp}:${restPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.region'='us-east-1' + ) + """ + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + + try { + sql """drop table if exists ${tableName}""" + sql """ + create table ${tableName} ( + id int not null, + required_name string not null, + optional_value int, + info struct, + attrs map + ) engine=iceberg + properties ('format-version'='2', 'write.format.default'='parquet') + """ + sql """ + insert into ${tableName} values + (1, 'name-1', 10, named_struct('value', 100, 'keep', 101), map('k', 1000)) + """ + sql """alter table ${tableName} create tag metadata_before_change""" + int initialSnapshots = snapshotCount() + + // Scenario S19-comment: top-level and nested comments are metadata-only schema changes. + sql """alter table ${tableName} modify column required_name comment 'top-level-comment'""" + sql """alter table ${tableName} modify column info.value comment 'nested-comment'""" + String descAfterComment = allDescText() + List> sparkDescription = spark_iceberg(""" + describe table extended demo.${dbName}.${tableName} + """) + String sparkDescriptionText = sparkDescription.flatten().collect { + it == null ? "" : it.toString() + }.join(" ") + assertTrue(sparkDescriptionText.contains("top-level-comment")) + // Negative contract: Doris DESC currently omits Iceberg field comments. + assertFalse(descAfterComment.contains("top-level-comment")) + assertFalse(descAfterComment.contains("nested-comment")) + assertEquals(initialSnapshots, snapshotCount()) + + // Scenario S19-nullability: relaxing required to optional preserves data and historical refs. + sql """alter table ${tableName} modify column required_name string null""" + assertEquals([[1, "name-1", 100]], sql(""" + select id, required_name, info.value from ${tableName} order by id + """)) + assertEquals([[1, "name-1", 100]], sql(""" + select id, required_name, info.value + from ${tableName}@tag(metadata_before_change) + order by id + """)) + assertEquals(initialSnapshots, snapshotCount()) + + // Scenario S20-required-add: Iceberg rejects non-null additions and commits no metadata. + int metadataBeforeRequiredAdd = schemaCount() + test { + sql """alter table ${tableName} add column required_added int not null""" + exception "default value" + } + assertEquals(metadataBeforeRequiredAdd, schemaCount()) + + // Scenario S20-default: v2 non-null initial defaults are unsupported and atomic. + int metadataBeforeDefault = schemaCount() + test { + sql """ + alter table ${tableName} + add column default_added string default 'default-value' + """ + exception "non-null default" + } + assertEquals(metadataBeforeDefault, schemaCount()) + + // Scenario S20-nullability-strengthen: optional fields cannot become required with old NULLs. + int metadataBeforeNotNull = schemaCount() + test { + sql """alter table ${tableName} modify column optional_value int not null""" + exception "not null" + } + assertEquals(metadataBeforeNotNull, schemaCount()) + + // Scenario S20-narrowing: illegal type narrowing is rejected without a schema commit. + int metadataBeforeNarrowing = schemaCount() + test { + sql """alter table ${tableName} modify column optional_value smallint""" + exception "not supported for Iceberg column" + } + assertEquals(metadataBeforeNarrowing, schemaCount()) + + // Scenario S20-map-key: map keys cannot be evolved independently. + int metadataBeforeMapKey = schemaCount() + test { + sql """alter table ${tableName} modify column attrs.`key` bigint""" + exception "Cannot modify MAP key nested column" + } + assertEquals(metadataBeforeMapKey, schemaCount()) + + sql """refresh table ${tableName}""" + assertEquals([[1, "name-1", 10, 100, 1000]], sql(""" + select id, required_name, optional_value, info.value, attrs['k'] + from ${tableName} + order by id + """)) + assertEquals(initialSnapshots, snapshotCount()) + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_position_dv_time_travel.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_position_dv_time_travel.groovy new file mode 100644 index 00000000000000..1afc70bc5f2074 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_position_dv_time_travel.groovy @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_schema_position_dv_time_travel", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_schema_position_dv_time_travel" + String dbName = "iceberg_schema_position_dv_db" + + def latestSnapshotId = { String tableName -> + return spark_iceberg(""" + select snapshot_id + from demo.${dbName}.${tableName}.snapshots + order by committed_at desc + limit 1 + """)[0][0].toString() + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri'='http://${externalEnvIp}:${restPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.region'='us-east-1' + ) + """ + + try { + spark_iceberg """create database if not exists demo.${dbName}""" + + [ + [table: "position_parquet", version: "2", format: "parquet"], + [table: "position_orc", version: "2", format: "orc"], + [table: "dv_parquet", version: "3", format: "parquet"], + [table: "dv_orc", version: "3", format: "orc"] + ].each { profile -> + String tableName = profile.table + spark_iceberg_multi """ + drop table if exists demo.${dbName}.${tableName}; + create table demo.${dbName}.${tableName} ( + id int, + old_name string, + victim string, + metric int, + info struct + ) using iceberg + tblproperties ( + 'format-version'='${profile.version}', + 'write.format.default'='${profile.format}', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + insert into demo.${dbName}.${tableName} + select /*+ COALESCE(1) */ id, old_name, victim, metric, info from values + (1, 'old-1', 'victim-1', 10, + named_struct('old_child', 100, 'keep', 101)), + (2, 'old-2', 'victim-2', 20, + named_struct('old_child', 200, 'keep', 201)), + (3, 'old-3', 'victim-3', 30, + named_struct('old_child', 300, 'keep', 301)) + as t(id, old_name, victim, metric, info); + """ + String beforeDelete = latestSnapshotId(tableName) + + // Scenario TC04-position/DV-before-change: delete visibility is snapshot-local. + spark_iceberg """ + delete from demo.${dbName}.${tableName} where id = 2 + """ + String afterFirstDelete = latestSnapshotId(tableName) + + spark_iceberg_multi """ + alter table demo.${dbName}.${tableName} rename column old_name to new_name; + alter table demo.${dbName}.${tableName} + rename column info.old_child to new_child; + alter table demo.${dbName}.${tableName} drop column victim; + alter table demo.${dbName}.${tableName} add column victim bigint; + alter table demo.${dbName}.${tableName} alter column metric type bigint; + insert into demo.${dbName}.${tableName} + (id, new_name, victim, metric, info) values + (4, 'new-4', 4000, 6000000000, + named_struct('new_child', 400, 'keep', 401)); + delete from demo.${dbName}.${tableName} where id = 3; + """ + String afterSchemaDelete = latestSnapshotId(tableName) + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + sql """refresh table ${tableName}""" + + assertEquals([ + [1, "old-1", "victim-1", 10, 100], + [2, "old-2", "victim-2", 20, 200], + [3, "old-3", "victim-3", 30, 300] + ], sql(""" + select id, old_name, victim, metric, info.old_child + from ${tableName} for version as of ${beforeDelete} + order by id + """)) + assertEquals([[1], [3]], sql(""" + select id + from ${tableName} for version as of ${afterFirstDelete} + order by id + """)) + + // Scenario TC04-S04/S06/S07/S08/S10: delete files survive top-level and nested evolution. + assertEquals([ + [1, "old-1", null, 10L, 100], + [4, "new-4", 4000L, 6000000000L, 400] + ], sql(""" + select id, new_name, victim, metric, info.new_child + from ${tableName} for version as of ${afterSchemaDelete} + order by id + """)) + + // Scenario TC04-field-ID: re-added victim never exposes values from the dropped STRING field. + assertEquals([[1, null], [4, 4000L]], sql(""" + select id, victim + from ${tableName} for version as of ${afterSchemaDelete} + order by id + """)) + + // Scenario TC04-reader-diff: both scanners apply position deletes or DVs identically. + sql """set enable_file_scanner_v2=true""" + List> v2Rows = sql(""" + select id, new_name, victim, metric, info.new_child + from ${tableName} for version as of ${afterSchemaDelete} + where metric >= 10 + order by id + """) + sql """set enable_file_scanner_v2=false""" + List> legacyRows = sql(""" + select id, new_name, victim, metric, info.new_child + from ${tableName} for version as of ${afterSchemaDelete} + where metric >= 10 + order by id + """) + assertEquals(v2Rows, legacyRows) + + // Scenario TC04-delete-file-kind: v2 produces position deletes and v3 produces DVs. + List> deleteFiles = spark_iceberg(""" + select content, file_format + from demo.${dbName}.${tableName}.delete_files + """) + assertTrue(deleteFiles.size() > 0, + "The ${profile.version == '3' ? 'DV' : 'position-delete'} profile needs delete files") + } + } finally { + sql """set enable_file_scanner_v2=true""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.groovy new file mode 100644 index 00000000000000..607895a6e21fb4 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_ref_actions_matrix.groovy @@ -0,0 +1,242 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_schema_ref_actions_matrix", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_schema_ref_actions_matrix" + String dbName = "iceberg_schema_ref_actions_db" + + def snapshots = { String tableName -> + return sql(""" + select snapshot_id + from ${tableName}\$snapshots + order by committed_at, snapshot_id + """).collect { row -> row[0].toString() } + } + + def assertUnknownColumn = { String query, String columnName -> + test { + sql query + exception "'${columnName}'" + } + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri'='http://${externalEnvIp}:${restPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.region'='us-east-1' + ) + """ + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + + try { + // Scenario T15: rollback_to_snapshot rolls data back while the current schema stays latest. + String rollbackTable = "rollback_schema_timeline" + sql """drop table if exists ${rollbackTable}""" + sql """ + create table ${rollbackTable} ( + id int, + old_name string, + payload struct + ) engine=iceberg + properties ('format-version'='2', 'write.format.default'='parquet') + """ + sql """ + insert into ${rollbackTable} + values (1, 'old-1', named_struct('old_child', 10, 'keep', 11)) + """ + String rollbackOldSnapshot = snapshots(rollbackTable).last() + sql """alter table ${rollbackTable} create tag rollback_old""" + + sql """alter table ${rollbackTable} rename column old_name new_name""" + sql """alter table ${rollbackTable} rename column payload.old_child new_child""" + sql """ + insert into ${rollbackTable} + values (2, 'new-2', named_struct('new_child', 20, 'keep', 21)) + """ + String rollbackNewSnapshot = snapshots(rollbackTable).last() + sql """alter table ${rollbackTable} create tag rollback_new""" + + assertEquals([[1, "old-1", 10]], sql(""" + select id, old_name, payload.old_child + from ${rollbackTable} for version as of ${rollbackOldSnapshot} + order by id + """)) + assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + select id, new_name, payload.new_child + from ${rollbackTable} for version as of ${rollbackNewSnapshot} + order by id + """)) + + sql """ + alter table ${rollbackTable} + execute rollback_to_snapshot('snapshot_id'='${rollbackOldSnapshot}') + """ + sql """refresh table ${rollbackTable}""" + assertEquals([[1, "old-1", 10]], sql(""" + select id, new_name, payload.new_child + from ${rollbackTable} + order by id + """)) + assertUnknownColumn("""select old_name from ${rollbackTable}""", "old_name") + assertEquals([[1, "old-1", 10]], sql(""" + select id, old_name, payload.old_child + from ${rollbackTable}@tag(rollback_old) + order by id + """)) + assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + select id, new_name, payload.new_child + from ${rollbackTable}@tag(rollback_new) + order by id + """)) + + // Scenario T16-cherrypick: append snapshot with the renamed schema can be replayed after rollback. + sql """ + alter table ${rollbackTable} + execute cherrypick_snapshot('snapshot_id'='${rollbackNewSnapshot}') + """ + sql """refresh table ${rollbackTable}""" + assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + select id, new_name, payload.new_child + from ${rollbackTable} + order by id + """)) + assertEquals([[1, "old-1", 10]], sql(""" + select id, old_name, payload.old_child + from ${rollbackTable}@tag(rollback_old) + order by id + """)) + + // Scenario T15-timestamp: timestamp rollback also keeps the latest current schema. + String timestampTable = "rollback_timestamp_schema_timeline" + sql """drop table if exists ${timestampTable}""" + sql """ + create table ${timestampTable} ( + id int, + old_name string + ) engine=iceberg + properties ('format-version'='2', 'write.format.default'='orc') + """ + sql """insert into ${timestampTable} values (1, 'old-1')""" + List> timestampCheckpoint = sql(""" + select snapshot_id, + date_format(date_add(committed_at, interval 1 second), + '%Y-%m-%d %H:%i:%s.000') + from ${timestampTable}\$snapshots + order by committed_at desc + limit 1 + """) + String timestampOldSnapshot = timestampCheckpoint[0][0].toString() + String rollbackTimestamp = timestampCheckpoint[0][1].toString() + sql """alter table ${timestampTable} create tag timestamp_old""" + Thread.sleep(1100) + sql """alter table ${timestampTable} rename column old_name new_name""" + sql """insert into ${timestampTable} values (2, 'new-2')""" + + sql """ + alter table ${timestampTable} + execute rollback_to_timestamp('timestamp'='${rollbackTimestamp}') + """ + sql """refresh table ${timestampTable}""" + assertEquals([[1, "old-1"]], sql(""" + select id, new_name from ${timestampTable} order by id + """)) + assertEquals([[1, "old-1"]], sql(""" + select id, old_name + from ${timestampTable} for version as of ${timestampOldSnapshot} + order by id + """)) + + // Scenario T16-fast-forward: advance a pre-rename branch to the renamed main schema. + String fastForwardTable = "fast_forward_schema_timeline" + sql """drop table if exists ${fastForwardTable}""" + sql """ + create table ${fastForwardTable} ( + id int, + old_name string, + metric int + ) engine=iceberg + properties ('format-version'='2', 'write.format.default'='parquet') + """ + sql """insert into ${fastForwardTable} values (1, 'old-1', 10)""" + sql """alter table ${fastForwardTable} create branch pre_rename_branch""" + sql """alter table ${fastForwardTable} create tag pre_rename_tag""" + sql """alter table ${fastForwardTable} rename column old_name new_name""" + sql """alter table ${fastForwardTable} modify column metric bigint""" + sql """insert into ${fastForwardTable} values (2, 'new-2', 6000000000)""" + + // Scenario T08 negative contract: before fast-forward, branch reads use the latest rename schema. + test { + sql """ + select id, old_name, metric + from ${fastForwardTable}@branch(pre_rename_branch) + order by id + """ + exception "Unknown column 'old_name'" + } + assertEquals([[1, "old-1", 10]], sql(""" + select id, old_name, metric + from ${fastForwardTable}@tag(pre_rename_tag) + order by id + """)) + + // Scenario T09 negative contract: a pre-rename branch write uses main's latest schema. + test { + sql """ + insert into ${fastForwardTable}@branch(pre_rename_branch) + (id, old_name, metric) values (3, 'branch-3', 30) + """ + exception "Unknown column 'old_name'" + } + + sql """ + alter table ${fastForwardTable} + execute fast_forward('branch'='pre_rename_branch', 'to'='main') + """ + sql """refresh table ${fastForwardTable}""" + assertEquals([[1, "old-1", 10L], [2, "new-2", 6000000000L]], sql(""" + select id, new_name, metric + from ${fastForwardTable}@branch(pre_rename_branch) + order by id + """)) + assertEquals([[1, "old-1", 10]], sql(""" + select id, old_name, metric + from ${fastForwardTable}@tag(pre_rename_tag) + order by id + """)) + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy new file mode 100644 index 00000000000000..e0876e17ea5833 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy @@ -0,0 +1,732 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_schema_time_travel_matrix", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_schema_time_travel_matrix" + String noCacheCatalogName = "test_iceberg_schema_time_travel_matrix_no_cache" + String dbName = "iceberg_schema_time_travel_matrix_db" + String topTable = "top_timeline" + String nestedTable = "nested_timeline" + String dorisNestedTable = "doris_nested_timeline" + String deleteTable = "delete_partition_timeline" + + def latestSnapshotId = { String tableName -> + List> rows = spark_iceberg """ + SELECT snapshot_id + FROM demo.${dbName}.${tableName}.snapshots + ORDER BY committed_at DESC + LIMIT 1 + """ + assertEquals(1, rows.size()) + return rows[0][0].toString() + } + + def assertUnknownColumn = { String query, String columnName -> + test { + sql query + exception "'${columnName}'" + } + } + + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${noCacheCatalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri'='http://${externalEnvIp}:${restPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.region'='us-east-1' + ) + """ + sql """ + CREATE CATALOG ${noCacheCatalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri'='http://${externalEnvIp}:${restPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.region'='us-east-1', + 'meta.cache.iceberg.table.ttl-second'='0', + 'meta.cache.iceberg.schema.ttl-second'='0' + ) + """ + + try { + spark_iceberg_multi """ + CREATE DATABASE IF NOT EXISTS demo.${dbName}; + DROP TABLE IF EXISTS demo.${dbName}.${topTable}; + CREATE TABLE demo.${dbName}.${topTable} ( + id INT, + old_name STRING, + victim STRING, + metric INT + ) USING iceberg + TBLPROPERTIES ( + 'format-version'='2', + 'write.format.default'='parquet' + ); + INSERT INTO demo.${dbName}.${topTable} + VALUES (1, 'alpha', 'old-v1', 10); + """ + String topCp0 = latestSnapshotId(topTable) + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${topTable}` CREATE TAG top_cp0""" + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${topTable}` CREATE BRANCH top_cp0_branch""" + Thread.sleep(1100) + + // Scenario S01/S02/S03 x T00/T01/T02/T05/T08: + // add and reorder columns while keeping the pre-change snapshot, tag and branch readable. + spark_iceberg_multi """ + ALTER TABLE demo.${dbName}.${topTable} + ADD COLUMNS (added STRING AFTER old_name, added_second BIGINT); + ALTER TABLE demo.${dbName}.${topTable} ALTER COLUMN added_second FIRST; + INSERT INTO demo.${dbName}.${topTable} + (id, old_name, victim, metric, added, added_second) + VALUES (2, 'beta', 'old-v2', 20, 'added-v2', 200); + """ + String topCpAdd = latestSnapshotId(topTable) + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${topTable}` CREATE TAG top_cp_add""" + Thread.sleep(1100) + + // Scenario S04/S05 x T00-T08: + // a rename must be observable through explicit old/new names, not only SELECT *. + spark_iceberg_multi """ + ALTER TABLE demo.${dbName}.${topTable} RENAME COLUMN old_name TO MixedName; + INSERT INTO demo.${dbName}.${topTable} + (id, MixedName, victim, metric, added, added_second) + VALUES (3, 'gamma', 'old-v3', 30, 'added-v3', 300); + """ + String topCpRename = latestSnapshotId(topTable) + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${topTable}` CREATE TAG top_cp_rename""" + Thread.sleep(1100) + + // Scenario S06 x T00/T01/T02/T05: old refs retain the dropped field. + spark_iceberg_multi """ + ALTER TABLE demo.${dbName}.${topTable} DROP COLUMN victim; + INSERT INTO demo.${dbName}.${topTable} + (id, MixedName, metric, added, added_second) + VALUES (4, 'delta', 40, 'added-v4', 400); + """ + String topCpDrop = latestSnapshotId(topTable) + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${topTable}` CREATE TAG top_cp_drop""" + Thread.sleep(1100) + + // Scenario S07 x T00/T01/T02/T05/T12/T13: + // reusing the name with a new BIGINT field ID must never expose old STRING values. + spark_iceberg_multi """ + ALTER TABLE demo.${dbName}.${topTable} ADD COLUMN victim BIGINT; + INSERT INTO demo.${dbName}.${topTable} + (id, MixedName, metric, added, added_second, victim) + VALUES (5, 'epsilon', 50, 'added-v5', 500, 5000); + """ + String topCpReadd = latestSnapshotId(topTable) + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${topTable}` CREATE TAG top_cp_readd""" + Thread.sleep(1100) + + // Scenario S08 x T00/T01/T02/T05: compatible INT -> BIGINT promotion. + spark_iceberg_multi """ + ALTER TABLE demo.${dbName}.${topTable} ALTER COLUMN metric TYPE BIGINT; + INSERT INTO demo.${dbName}.${topTable} + (id, MixedName, metric, added, added_second, victim) + VALUES (6, 'zeta', 6000000000, 'added-v6', 600, 6000); + """ + String topCpPromote = latestSnapshotId(topTable) + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${topTable}` CREATE TAG top_cp_promote""" + + spark_iceberg_multi """ + DROP TABLE IF EXISTS demo.${dbName}.${nestedTable}; + CREATE TABLE demo.${dbName}.${nestedTable} ( + id INT, + payload STRUCT, + attributes MAP>, + events ARRAY> + ) USING iceberg + TBLPROPERTIES ('format-version'='2', 'write.format.default'='orc'); + INSERT INTO demo.${dbName}.${nestedTable} VALUES ( + 1, + named_struct('old_child', 10, 'keep', 11), + map('a', named_struct('old_child', 20, 'keep', 21)), + array(named_struct('old_child', 30, 'keep', 31)) + ); + """ + String nestedCp0 = latestSnapshotId(nestedTable) + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${nestedTable}` CREATE TAG nested_cp0""" + + // Scenario S09/S14/S15 x T00/T01/T02/T05: + // add children to STRUCT, MAP value struct and ARRAY element struct. + spark_iceberg_multi """ + ALTER TABLE demo.${dbName}.${nestedTable} ADD COLUMN payload.added_child INT; + ALTER TABLE demo.${dbName}.${nestedTable} ADD COLUMN attributes.value.added_child INT; + ALTER TABLE demo.${dbName}.${nestedTable} ADD COLUMN events.element.added_child INT; + INSERT INTO demo.${dbName}.${nestedTable} VALUES ( + 2, + named_struct('old_child', 110, 'keep', 111, 'added_child', 112), + map('a', named_struct('old_child', 120, 'keep', 121, 'added_child', 122)), + array(named_struct('old_child', 130, 'keep', 131, 'added_child', 132)) + ); + """ + String nestedCpAdd = latestSnapshotId(nestedTable) + + // Scenario S10/S14/S15 x T00/T01/T02/T05: + // rename nested children and verify both positive and negative binding. + spark_iceberg_multi """ + ALTER TABLE demo.${dbName}.${nestedTable} + RENAME COLUMN payload.old_child TO renamed_child; + ALTER TABLE demo.${dbName}.${nestedTable} + RENAME COLUMN attributes.value.old_child TO renamed_child; + ALTER TABLE demo.${dbName}.${nestedTable} + RENAME COLUMN events.element.old_child TO renamed_child; + INSERT INTO demo.${dbName}.${nestedTable} VALUES ( + 3, + named_struct('renamed_child', 210, 'keep', 211, 'added_child', 212), + map('a', named_struct('renamed_child', 220, 'keep', 221, 'added_child', 222)), + array(named_struct('renamed_child', 230, 'keep', 231, 'added_child', 232)) + ); + """ + String nestedCpRename = latestSnapshotId(nestedTable) + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${nestedTable}` CREATE TAG nested_cp_rename""" + + // Scenario S11/S12/S13 x T00/T01/T02/T05: + // drop/re-add a nested name with a new ID and promote the surviving child type. + spark_iceberg_multi """ + ALTER TABLE demo.${dbName}.${nestedTable} DROP COLUMN payload.renamed_child; + ALTER TABLE demo.${dbName}.${nestedTable} DROP COLUMN attributes.value.renamed_child; + ALTER TABLE demo.${dbName}.${nestedTable} DROP COLUMN events.element.renamed_child; + ALTER TABLE demo.${dbName}.${nestedTable} ADD COLUMN payload.renamed_child BIGINT; + ALTER TABLE demo.${dbName}.${nestedTable} ADD COLUMN attributes.value.renamed_child BIGINT; + ALTER TABLE demo.${dbName}.${nestedTable} ADD COLUMN events.element.renamed_child BIGINT; + ALTER TABLE demo.${dbName}.${nestedTable} ALTER COLUMN payload.keep TYPE BIGINT; + INSERT INTO demo.${dbName}.${nestedTable} VALUES ( + 4, + named_struct('keep', 311, 'added_child', 312, 'renamed_child', 3100), + map('a', named_struct('keep', 321, 'added_child', 322, 'renamed_child', 3200)), + array(named_struct('keep', 331, 'added_child', 332, 'renamed_child', 3300)) + ); + """ + String nestedCpReadd = latestSnapshotId(nestedTable) + + spark_iceberg_multi """ + SET spark.sql.shuffle.partitions=1; + DROP TABLE IF EXISTS demo.${dbName}.${deleteTable}; + CREATE TABLE demo.${dbName}.${deleteTable} ( + id INT, + old_name STRING, + category STRING, + event_time TIMESTAMP + ) USING iceberg + PARTITIONED BY (days(event_time)) + TBLPROPERTIES ( + 'format-version'='2', + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'write.distribution-mode'='none', + 'write.target-file-size-bytes'='134217728' + ); + INSERT INTO demo.${dbName}.${deleteTable} + SELECT /*+ COALESCE(1) */ id, old_name, category, event_time FROM VALUES + (1, 'a', 'x', TIMESTAMP '2026-01-01 01:00:00'), + (2, 'b', 'x', TIMESTAMP '2026-01-01 02:00:00'), + (3, 'c', 'y', TIMESTAMP '2026-01-02 01:00:00') + AS t(id, old_name, category, event_time); + """ + String deleteCp0 = latestSnapshotId(deleteTable) + sql """ALTER TABLE `${catalogName}`.`${dbName}`.`${deleteTable}` CREATE TAG delete_cp0""" + + // Scenario S04/S17 x TC03/TC04: + // position deletes after rename and partition-spec evolution must not affect the old ref. + spark_iceberg_multi """ + ALTER TABLE demo.${dbName}.${deleteTable} RENAME COLUMN old_name TO new_name; + ALTER TABLE demo.${dbName}.${deleteTable} ADD PARTITION FIELD bucket(2, id); + INSERT INTO demo.${dbName}.${deleteTable} VALUES + (4, 'd', 'y', TIMESTAMP '2026-01-02 02:00:00'); + DELETE FROM demo.${dbName}.${deleteTable} WHERE id = 2; + """ + String deleteCpAfter = latestSnapshotId(deleteTable) + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + sql """refresh catalog ${catalogName}""" + + // Scenario S09-S15 x T00-T13 x TC02/TC04: + // exercise the nested Iceberg DDL implemented on the latest master through Doris itself, + // then combine it with snapshots, tags, a branch, dual-snapshot binding and a delete. + sql """drop table if exists ${dorisNestedTable}""" + sql """ + create table ${dorisNestedTable} ( + id int, + info struct, + events array>, + attrs map> + ) + """ + sql """ + insert into ${dorisNestedTable} values ( + 1, + struct(10, 'old-info'), + array(struct(100)), + map('k', struct(1000)) + ) + """ + String dorisNestedCp0 = sql(""" + select snapshot_id + from ${dorisNestedTable}\$snapshots + order by committed_at desc + limit 1 + """)[0][0].toString() + sql """ + alter table ${dorisNestedTable} + create tag doris_nested_cp0 as of version ${dorisNestedCp0} + """ + sql """ + alter table ${dorisNestedTable} + create branch doris_nested_cp0_branch as of version ${dorisNestedCp0} + """ + Thread.sleep(1100) + + // Scenario S09/S14/S15: Doris adds fields at nested STRUCT, ARRAY element and MAP value paths. + sql """ + alter table ${dorisNestedTable} + add column info.added string null after metric + """ + sql """ + alter table ${dorisNestedTable} + add column events.element.added int null after score + """ + sql """ + alter table ${dorisNestedTable} + add column attrs.value.added int null after code + """ + sql """ + insert into ${dorisNestedTable} values ( + 2, + struct(20, 'info-added', 'after-add'), + array(struct(200, 201)), + map('k', struct(2000, 2001)) + ) + """ + String dorisNestedCpAdd = sql(""" + select snapshot_id + from ${dorisNestedTable}\$snapshots + order by committed_at desc + limit 1 + """)[0][0].toString() + sql """ + alter table ${dorisNestedTable} + create tag doris_nested_cp_add as of version ${dorisNestedCpAdd} + """ + Thread.sleep(1100) + + // Scenario S10/S13/S14/S15: rename nested fields and promote sibling field types in one timeline. + sql """alter table ${dorisNestedTable} rename column info.added to renamed""" + sql """ + alter table ${dorisNestedTable} + rename column events.element.added to renamed + """ + sql """ + alter table ${dorisNestedTable} + rename column attrs.value.added to renamed + """ + sql """alter table ${dorisNestedTable} modify column info.metric bigint""" + sql """ + alter table ${dorisNestedTable} + modify column events.element.score bigint + """ + sql """ + alter table ${dorisNestedTable} + modify column attrs.value.code bigint + """ + sql """ + insert into ${dorisNestedTable} values ( + 3, + struct(cast(30 as bigint), 'info-renamed', 'after-rename'), + array(struct(cast(300 as bigint), 301)), + map('k', struct(cast(3000 as bigint), 3001)) + ) + """ + String dorisNestedCpRename = sql(""" + select snapshot_id + from ${dorisNestedTable}\$snapshots + order by committed_at desc + limit 1 + """)[0][0].toString() + sql """ + alter table ${dorisNestedTable} + create tag doris_nested_cp_rename as of version ${dorisNestedCpRename} + """ + Thread.sleep(1100) + + // Scenario S11/S12/S14/S15: re-add the same nested names with new BIGINT field IDs. + sql """alter table ${dorisNestedTable} drop column info.renamed""" + sql """alter table ${dorisNestedTable} drop column events.element.renamed""" + sql """alter table ${dorisNestedTable} drop column attrs.value.renamed""" + sql """alter table ${dorisNestedTable} add column info.renamed bigint null""" + sql """ + alter table ${dorisNestedTable} + add column events.element.renamed bigint null + """ + sql """ + alter table ${dorisNestedTable} + add column attrs.value.renamed bigint null + """ + sql """ + insert into ${dorisNestedTable} values ( + 4, + struct(cast(40 as bigint), 'info-readd', cast(4001 as bigint)), + array(struct(cast(400 as bigint), cast(401 as bigint))), + map('k', struct(cast(4000 as bigint), cast(4001 as bigint))) + ) + """ + String dorisNestedCpReadd = sql(""" + select snapshot_id + from ${dorisNestedTable}\$snapshots + order by committed_at desc + limit 1 + """)[0][0].toString() + sql """ + alter table ${dorisNestedTable} + create tag doris_nested_cp_readd as of version ${dorisNestedCpReadd} + """ + + // Scenario TC04: a delete after nested evolution must not change any older snapshot/tag. + sql """delete from ${dorisNestedTable} where id = 2""" + String dorisNestedCpDelete = sql(""" + select snapshot_id + from ${dorisNestedTable}\$snapshots + order by committed_at desc + limit 1 + """)[0][0].toString() + + // Scenario TC02/T01/T05/T06/T08: every old reference exposes its own nested schema. + assertEquals([[1, 10, 100, 1000]], + sql(""" + select id, info.metric, events[1].score, attrs['k'].code + from ${dorisNestedTable} for version as of ${dorisNestedCp0} + order by id + """)) + assertEquals([[1, 10, 100, 1000]], + sql(""" + select id, info.metric, events[1].score, attrs['k'].code + from ${dorisNestedTable} for version as of 'doris_nested_cp0' + order by id + """)) + assertEquals([[1, 10, 100, 1000]], + sql(""" + select id, info.metric, events[1].score, attrs['k'].code + from ${dorisNestedTable}@tag(doris_nested_cp0) + order by id + """)) + // Negative contract: an old branch currently leaks the latest BIGINT nested types. + assertEquals([[1, 10L, 100L, 1000L]], + sql(""" + select id, info.metric, events[1].score, attrs['k'].code + from ${dorisNestedTable}@branch(doris_nested_cp0_branch) + order by id + """)) + assertUnknownColumn(""" + select info.renamed + from ${dorisNestedTable} for version as of ${dorisNestedCp0} + """, "renamed") + + // Scenario TC02/T03/T04: complex-field time travel uses the pre-change nested schema. + List> dorisNestedCp0Time = sql(""" + select date_format(date_add(committed_at, interval 1 second), '%Y-%m-%d %H:%i:%s'), + cast(unix_timestamp(committed_at) * 1000 + 999 as bigint) + from ${dorisNestedTable}\$snapshots + where snapshot_id = ${dorisNestedCp0} + """) + assertEquals([[1, 10]], + sql(""" + select id, info.metric + from ${dorisNestedTable} + for time as of "${dorisNestedCp0Time[0][0]}" + order by id + """)) + // Scenario TC03-negative: Iceberg accepts time strings, not Paimon-style epoch millis. + test { + sql """ + select id, info.metric + from ${dorisNestedTable} + for time as of ${dorisNestedCp0Time[0][1]} + order by id + """ + exception "can't parse time" + } + + // Scenario TC02: nested add/rename/drop-readd checkpoints verify projection and predicates. + assertEquals([[1, null, null, null], [2, "info-added", 201, 2001]], + sql(""" + select id, info.added, events[1].added, attrs['k'].added + from ${dorisNestedTable} for version as of ${dorisNestedCpAdd} + order by id + """)) + assertEquals([[1, null, null, null], [2, "info-added", 201, 2001], + [3, "info-renamed", 301, 3001]], + sql(""" + select id, info.renamed, events[1].renamed, attrs['k'].renamed + from ${dorisNestedTable} for version as of ${dorisNestedCpRename} + where info.metric >= 10 + order by id + """)) + assertUnknownColumn(""" + select info.added + from ${dorisNestedTable} for version as of ${dorisNestedCpRename} + """, "added") + assertEquals([[1, null, null, null], [2, null, null, null], + [3, null, null, null], [4, 4001L, 401L, 4001L]], + sql(""" + select id, info.renamed, events[1].renamed, attrs['k'].renamed + from ${dorisNestedTable} for version as of ${dorisNestedCpReadd} + order by id + """)) + + // Scenario TC04: delete is visible only at/after the delete snapshot. + assertEquals([1, 2, 3, 4], + sql(""" + select id + from ${dorisNestedTable} for version as of ${dorisNestedCpReadd} + order by id + """).collect { it[0] }) + assertEquals([1, 3, 4], + sql(""" + select id + from ${dorisNestedTable} for version as of ${dorisNestedCpDelete} + order by id + """).collect { it[0] }) + + // Scenario TC01: current schema uses only the new names and promoted/re-added types. + List currentColumns = sql("""desc ${topTable}""").collect { it[0].toString() } + assertEquals(["added_second", "id", "MixedName", "added", "metric", "victim"], currentColumns) + assertEquals([[1, null], [2, null], [3, null], [4, null], [5, 5000L], [6, 6000L]], + sql("""select id, victim from ${topTable} order by id""")) + assertEquals([[6, 6000000000L]], + sql("""select id, metric from ${topTable} where metric > 5000000000 order by id""")) + assertUnknownColumn("""select old_name from ${topTable}""", "old_name") + + // Scenario T01/T05/T06/T08: the pre-change snapshot/tag/branch binds old_name and victim. + List> topCp0Rows = [[1, "alpha", "old-v1", 10]] + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of ${topCp0} + order by id + """)) + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of 'top_cp0' + order by id + """)) + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable}@tag(top_cp0) + order by id + """)) + // Negative contract: an old branch is currently analyzed with the latest rename schema. + test { + sql """ + select id, old_name, victim, metric + from ${topTable}@branch(top_cp0_branch) + order by id + """ + exception "Unknown column 'old_name'" + } + assertUnknownColumn(""" + select MixedName from ${topTable} for version as of ${topCp0} + """, "MixedName") + + // Scenario T03/T04: validate string time travel and reject unsupported epoch millis. + List> cp0TimeRows = sql(""" + select date_format(date_add(committed_at, interval 1 second), '%Y-%m-%d %H:%i:%s'), + cast(unix_timestamp(committed_at) * 1000 + 999 as bigint) + from ${topTable}\$snapshots + where snapshot_id = ${topCp0} + """) + assertEquals(1, cp0TimeRows.size()) + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable} for time as of "${cp0TimeRows[0][0]}" + order by id + """)) + test { + sql """ + select id, old_name, victim, metric + from ${topTable} for time as of ${cp0TimeRows[0][1]} + order by id + """ + exception "can't parse time" + } + + // Scenario S01-S05: every checkpoint verifies projection, predicate and aggregation. + assertEquals([[1, "alpha", null], [2, "beta", "added-v2"]], + sql(""" + select id, old_name, added + from ${topTable} for version as of ${topCpAdd} + where metric >= 10 + order by id + """)) + assertEquals([[3L, 60L]], + sql(""" + select count(*), sum(metric) + from ${topTable} for version as of ${topCpRename} + """)) + assertUnknownColumn(""" + select old_name from ${topTable} for version as of ${topCpRename} + """, "old_name") + assertEquals([[1, "old-v1"], [2, "old-v2"], [3, "old-v3"]], + sql(""" + select id, victim + from ${topTable} for version as of 'top_cp_rename' + where victim like 'old-%' + order by id + """)) + assertUnknownColumn(""" + select victim from ${topTable} for version as of ${topCpDrop} + """, "victim") + assertEquals([[1, null], [2, null], [3, null], [4, null], [5, 5000L]], + sql(""" + select id, victim + from ${topTable} for version as of ${topCpReadd} + order by id + """)) + assertEquals([[6, 6000000000L]], + sql(""" + select id, metric + from ${topTable} for version as of ${topCpPromote} + where metric > 5000000000 + """)) + + // Scenario TC02: nested projection/predicate use each snapshot's nested field IDs. + assertEquals([[1, 10, 20, 30]], + sql(""" + select id, payload.old_child, + element_at(attributes, 'a').old_child, + events[1].old_child + from ${nestedTable} for version as of ${nestedCp0} + where payload.old_child = 10 + """)) + assertEquals([[1, null, null, null], [2, 112, 122, 132]], + sql(""" + select id, payload.added_child, + element_at(attributes, 'a').added_child, + events[1].added_child + from ${nestedTable} for version as of ${nestedCpAdd} + order by id + """)) + assertEquals([[1, 10, 20, 30], [2, 110, 120, 130], [3, 210, 220, 230]], + sql(""" + select id, payload.renamed_child, + element_at(attributes, 'a').renamed_child, + events[1].renamed_child + from ${nestedTable} for version as of 'nested_cp_rename' + order by id + """)) + assertUnknownColumn(""" + select payload.old_child + from ${nestedTable} for version as of ${nestedCpRename} + """, "old_child") + assertEquals([[1, null], [2, null], [3, null], [4, 3100L]], + sql(""" + select id, payload.renamed_child + from ${nestedTable} for version as of ${nestedCpReadd} + order by id + """)) + + // Scenario TC03/TC04: delete visibility and renamed delete-table fields are snapshot-local. + assertEquals([[1, "a"], [2, "b"], [3, "c"]], + sql(""" + select id, old_name + from ${deleteTable} for version as of 'delete_cp0' + order by id + """)) + assertEquals([[1, "a"], [3, "c"], [4, "d"]], + sql("""select id, new_name from ${deleteTable} order by id""")) + assertEquals([[1, "a"], [3, "c"], [4, "d"]], + sql(""" + select id, new_name + from ${deleteTable} for version as of ${deleteCpAfter} + order by id + """)) + assertTrue(((Number) sql(""" + select count(*) from ${deleteTable}\$position_deletes + """)[0][0]).longValue() > 0L) + + // Scenario TC08/S20: illegal narrowing and dropping a partition source are atomic failures. + test { + sql """alter table ${topTable} modify column metric int""" + exception "Cannot" + } + test { + sql """alter table ${deleteTable} drop column event_time""" + exception "Cannot" + } + assertEquals([[6, 6000000000L]], + sql("""select id, metric from ${topTable} where id = 6""")) + assertEquals([[1, "a"], [3, "c"], [4, "d"]], + sql("""select id, new_name from ${deleteTable} order by id""")) + + // Scenario TC09/R13/R17: cache-off and scanner V1/V2 must produce identical historical rows. + sql """switch ${noCacheCatalogName}""" + sql """use ${dbName}""" + List> noCacheRows = sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of ${topCp0} + order by id + """) + assertEquals(topCp0Rows, noCacheRows) + sql """switch ${catalogName}""" + sql """use ${dbName}""" + sql """set enable_file_scanner_v2=false""" + List> legacyRows = sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of ${topCp0} + order by id + """) + sql """set enable_file_scanner_v2=true""" + List> v2Rows = sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of ${topCp0} + order by id + """) + assertEquals(legacyRows, v2Rows) + + // Scenario T11: an unknown snapshot/tag must fail instead of silently reading latest. + test { + sql """select * from ${topTable} for version as of 9223372036854775807""" + exception "does not have snapshotId 9223372036854775807" + } + test { + sql """select * from ${topTable} for version as of 'missing_schema_tag'""" + exception "does not have tag or branch named missing_schema_tag" + } + } finally { + sql """set enable_file_scanner_v2=false""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${noCacheCatalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_system_table_projection.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_system_table_projection.groovy new file mode 100644 index 00000000000000..a7d5acf75963f6 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_system_table_projection.groovy @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_system_table_projection", "p0,external,iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String catalogName = "test_iceberg_system_table_projection" + String dbName = "test_iceberg_system_table_projection_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${restPort}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + );""" + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + + def verifySystemTableProjection = { String tableName, String writeFormat -> + sql """DROP TABLE IF EXISTS ${tableName}""" + sql """ + CREATE TABLE ${tableName} ( + id int, + t_map_boolean map, + dt int + ) ENGINE=iceberg + PARTITION BY LIST (dt) () + PROPERTIES ( + "write-format" = "${writeFormat}" + ); + """ + sql """ + INSERT INTO ${tableName} + VALUES + (1, MAP(true, false), 20260702); + """ + + List> dataFiles = sql """ + SELECT file_size_in_bytes + FROM ${tableName}\$data_files + ORDER BY file_size_in_bytes; + """ + assertEquals(1, dataFiles.size()) + assertTrue(((Number) dataFiles[0][0]).longValue() > 0) + + List> filesSize = sql """ + SELECT file_size_in_bytes + FROM ${tableName}\$files + ORDER BY file_size_in_bytes; + """ + assertEquals(1, filesSize.size()) + assertTrue(((Number) filesSize[0][0]).longValue() > 0) + + List> files = sql """ + SELECT file_path, record_count + FROM ${tableName}\$files + ORDER BY file_path; + """ + assertEquals(1, files.size()) + assertTrue(files[0][0].toString().contains(tableName)) + assertEquals(1L, ((Number) files[0][1]).longValue()) + + List> snapshots = sql """ + SELECT snapshot_id, parent_id, operation + FROM ${tableName}\$snapshots + ORDER BY committed_at; + """ + assertEquals(1, snapshots.size()) + long snapshotId = ((Number) snapshots[0][0]).longValue() + assertTrue(snapshotId > 0) + assertEquals(null, snapshots[0][1]) + assertEquals("append", snapshots[0][2]) + + List> history = sql """ + SELECT snapshot_id, parent_id, is_current_ancestor + FROM ${tableName}\$history + ORDER BY made_current_at; + """ + assertEquals(1, history.size()) + assertEquals(snapshotId, ((Number) history[0][0]).longValue()) + assertEquals(null, history[0][1]) + assertEquals(true, history[0][2]) + } + + def verifyReadableMetricsProjection = { String tableName, String writeFormat -> + sql """DROP TABLE IF EXISTS ${tableName}""" + sql """ + CREATE TABLE ${tableName} ( + id int, + name string, + dt int + ) ENGINE=iceberg + PARTITION BY LIST (dt) () + PROPERTIES ( + "write-format" = "${writeFormat}" + ); + """ + sql """ + INSERT INTO ${tableName} + VALUES + (1, 'alice', 20260702), + (2, 'bob', 20260702); + """ + + List> files = sql """ + SELECT readable_metrics + FROM ${tableName}\$files + ORDER BY file_path; + """ + assertEquals(1, files.size()) + String readableMetrics = files[0][0].toString() + assertTrue(readableMetrics.contains("\"id\"")) + assertTrue(readableMetrics.contains("\"name\"")) + assertTrue(readableMetrics.contains("\"lower_bound\"")) + assertTrue(readableMetrics.contains("\"upper_bound\"")) + + // `name` is not the first field of readable_metrics. Keep nested column pruning enabled + // to verify that the system table reader preserves the SDK struct field ordinals. + List> nameUpperBound = sql """ + SELECT /*+ SET_VAR(enable_prune_nested_column=true) */ readable_metrics.name.upper_bound + FROM ${tableName}\$files + ORDER BY file_path; + """ + assertEquals(1, nameUpperBound.size()) + assertEquals("bob", nameUpperBound[0][0]) + } + + verifySystemTableProjection("test_iceberg_system_table_projection_orc", "orc") + verifySystemTableProjection("test_iceberg_system_table_projection_parquet", "parquet") + + test { + sql """SELECT COUNT(*) FROM test_iceberg_system_table_projection_orc\$data_files""" + result([[1L]]) + } + test { + sql """SELECT COUNT(*) FROM test_iceberg_system_table_projection_orc\$files""" + result([[1L]]) + } + test { + sql """SELECT COUNT(*) FROM test_iceberg_system_table_projection_parquet\$data_files""" + result([[1L]]) + } + test { + sql """SELECT COUNT(*) FROM test_iceberg_system_table_projection_parquet\$files""" + result([[1L]]) + } + + verifyReadableMetricsProjection("test_iceberg_system_table_projection_readable_metrics", "orc") +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v2_to_v3_doris_spark_compare.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v2_to_v3_doris_spark_compare.groovy index df6d1bbea20087..5d6451d9006120 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v2_to_v3_doris_spark_compare.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v2_to_v3_doris_spark_compare.groovy @@ -21,7 +21,6 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external logger.info("Iceberg test is disabled") return } - def catalogName = "test_iceberg_v2_to_v3_doris_spark_compare" def dbName = "test_v2_to_v3_doris_spark_compare_db" def restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") @@ -31,7 +30,100 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external def formats = ["parquet", "orc"] def tableNameForFormat = { baseName, format -> - return format == "parquet" ? baseName : "${baseName}_orc" + return "${baseName}_${format}" + } + + def createSparkUpgradeFixture = { tableName, format, partitioned, datePrefix, deleteBeforeUpgrade -> + def partitionClause = partitioned ? "partitioned by (days(dt))" : "" + def deleteClause = deleteBeforeUpgrade ? """ + delete from ${tableName} + where id = 2; + """ : "" + spark_iceberg_multi """ + use demo.${dbName}; + + drop table if exists ${tableName}; + + create table ${tableName} ( + id int, + tag string, + score int, + dt date + ) using iceberg + ${partitionClause} + tblproperties ( + 'format-version' = '2', + 'write.format.default' = '${format}', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ); + + insert into ${tableName} values + (1, 'base', 100, date '${datePrefix}-01'), + (2, 'base', 200, date '${datePrefix}-02'); + + insert into ${tableName} values + (3, 'base', 300, date '${datePrefix}-03'); + + update ${tableName} + set tag = 'base_u', score = score + 10 + where id = 1; + + ${deleteClause} + alter table ${tableName} + set tblproperties ('format-version' = '3'); + """ + } + + def createSparkReferenceFixture = { tableName, format -> + createSparkUpgradeFixture(tableName, format, true, "2024-02", false) + spark_iceberg_multi """ + use demo.${dbName}; + + update ${tableName} + set tag = 'post_v3_u', score = score + 20 + where id = 2; + + insert into ${tableName} values + (4, 'post_v3_i', 400, date '2024-02-04'); + + call demo.system.rewrite_data_files( + table => 'demo.${dbName}.${tableName}', + options => map('target-file-size-bytes', '10485760', 'min-input-files', '1') + ); + """ + } + + spark_iceberg """create database if not exists demo.${dbName}""" + formats.each { format -> + createSparkUpgradeFixture( + tableNameForFormat("v2v3_row_lineage_null_after_upgrade", format), + format, + true, + "2024-01", + false) + createSparkReferenceFixture(tableNameForFormat("v2v3_spark_ops_reference", format), format) + createSparkUpgradeFixture( + tableNameForFormat("v2v3_doris_ops_target", format), + format, + true, + "2024-02", + false) + (1..5).each { caseIndex -> + createSparkUpgradeFixture( + tableNameForFormat("v2v3_doris_upd_case${caseIndex}", format), + format, + true, + "2024-01", + true) + createSparkUpgradeFixture( + tableNameForFormat("v2v3_doris_unpart_case${caseIndex}", format), + format, + false, + "2024-01", + true) + } } sql """drop catalog if exists ${catalogName}""" @@ -53,35 +145,90 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external try { def assertV2RowsAreNullAfterUpgrade = { tableName -> + sql """refresh table ${dbName}.${tableName}""" def rows = sql """ select id, _row_id, _last_updated_sequence_number from ${tableName} order by id """ + if (rows.size() != 2) { + def dorisBusinessRows = sql """ + select id, tag, score, dt + from ${tableName} + order by id, tag, score + """ + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkBusinessRows = spark_iceberg """ + select id, tag, score, dt + from demo.${dbName}.${tableName} + order by id, tag, score + """ + log.info("Unexpected v2-to-v3 precheck row count for ${tableName}: lineageRows=${rows}, " + "dorisBusinessRows=${dorisBusinessRows}, sparkBusinessRows=${sparkBusinessRows}") + } assertEquals(2, rows.size()) rows.each { row -> assertTrue(row[1] == null, - "_row_id should be null for v2 rows after upgrade in ${tableName}, row=${row}") + "_row_id should be null for v2 rows after upgrade in ${tableName}, row=${row}") assertTrue(row[2] == null, - "_last_updated_sequence_number should be null for v2 rows after upgrade in ${tableName}, row=${row}") + "_last_updated_sequence_number should be null for v2 rows after upgrade in ${tableName}, row=${row}") } } - def assertV23RowsNotNullAfterUpd = { tableName -> + def assertLineageState = { tableName, nonNullIds, nullIds -> def rows = sql """ select id, _row_id, _last_updated_sequence_number from ${tableName} order by id """ - rows.each { row -> + log.info("Lineage state for ${tableName}: ${rows}") + Map> rowsById = [:] + rows.each { row -> rowsById[row[0].toString().toInteger()] = row + } + nonNullIds.each { id -> + assertTrue(rowsById.containsKey(id), "id=${id} should exist in ${tableName}, rows=${rows}") + def row = rowsById[id] assertTrue(row[1] != null, - "_row_id should be non-null after Doris operator for ${tableName}") + "_row_id should be non-null after Doris operator for ${tableName}, row=${row}") assertTrue(row[2] != null, - "_last_updated_sequence_number should be non-null after Doris operator for ${tableName}") + "_last_updated_sequence_number should be non-null after Doris operator for ${tableName}, row=${row}") + } + nullIds.each { id -> + assertTrue(rowsById.containsKey(id), "id=${id} should exist in ${tableName}, rows=${rows}") + def row = rowsById[id] + assertTrue(row[1] == null, + "_row_id should stay null for historical v2 row in ${tableName}, row=${row}") + assertTrue(row[2] == null, + "_last_updated_sequence_number should stay null for historical v2 row in ${tableName}, row=${row}") + } + } + def normalizeBusinessRows = { rows -> + return rows.collect { row -> + [row[0].toString().toInteger(), + row[1].toString(), + row[2].toString().toInteger(), + row[3].toString()] } } + def assertSparkBusinessRowsEqual = { tableName -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg """ + select id, tag, score, dt + from demo.${dbName}.${tableName} + order by id + """ + def dorisRows = sql """ + select id, tag, score, dt + from ${tableName} + order by id + """ + log.info("Doris business rows for ${tableName}: ${dorisRows}") + log.info("Spark business rows for ${tableName}: ${sparkRows}") + assertEquals(normalizeBusinessRows(dorisRows), normalizeBusinessRows(sparkRows)) + } + def upgradeV3DorisOperationInsert = { tableName -> assertV2RowsAreNullAfterUpgrade(tableName) @@ -98,7 +245,8 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external assertEquals(3, rows.size()) assertEquals(4, rows[2][0].toString().toInteger()) assertEquals("post_v3_i", rows[2][1]) - assertV23RowsNotNullAfterUpd(tableName) + assertLineageState(tableName, [1, 3, 4], []) + assertSparkBusinessRowsEqual(tableName) } def upgradeV3DorisOperationDelete = { tableName -> @@ -116,7 +264,8 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external """ assertEquals(1, rows.size()) assertEquals(1, rows[0][0].toString().toInteger()) - assertV23RowsNotNullAfterUpd(tableName) + assertLineageState(tableName, [1], []) + assertSparkBusinessRowsEqual(tableName) } @@ -137,7 +286,8 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external assertEquals(2, rows.size()) assertEquals(1, rows[0][0].toString().toInteger()) assertEquals("post_v3_u", rows[0][1]) - assertV23RowsNotNullAfterUpd(tableName) + assertLineageState(tableName, [1, 3], []) + assertSparkBusinessRowsEqual(tableName) } def upgradeV3DorisOperationRewrite = { tableName -> @@ -151,14 +301,59 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external ) """) assertTrue(rewriteResult.size() > 0, - "rewrite_data_files should return summary rows for ${tableName}") + "rewrite_data_files should return summary rows for ${tableName}") + sql """refresh table ${dbName}.${tableName}""" def rowCount = sql """ select count(*) from ${tableName} """ assertEquals(2, rowCount[0][0].toString().toInteger()) - assertV23RowsNotNullAfterUpd(tableName) + assertLineageState(tableName, [1, 3], []) + assertSparkBusinessRowsEqual(tableName) + } + + def validateV3UpgradeReadCompatibility = { tableName -> + assertV2RowsAreNullAfterUpgrade(tableName) + assertSparkBusinessRowsEqual(tableName) + } + + def upgradeV3DorisOperationMerge = { tableName -> + assertV2RowsAreNullAfterUpgrade(tableName) + + sql """ + merge into ${tableName} t + using ( + select 1 as id, 'post_v3_m_u' as tag, 101 as score, date '2024-01-01' as dt, 'U' as flag + union all + select 3, 'base3', 30, date '2024-01-03', 'D' + union all + select 4, 'post_v3_m_i', 40, date '2024-01-04', 'I' + ) s + on t.id = s.id + when matched and s.flag = 'D' then delete + when matched then update set tag = s.tag, score = s.score + when not matched then insert (id, tag, score, dt) + values (s.id, s.tag, s.score, s.dt) + """ + + def rows = sql """ + select id, tag, score, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """ + assertEquals(2, rows.size()) + assertEquals(1, rows[0][0].toString().toInteger()) + assertEquals("post_v3_m_u", rows[0][1].toString()) + assertEquals(101, rows[0][2].toString().toInteger()) + assertEquals(4, rows[1][0].toString().toInteger()) + assertEquals("post_v3_m_i", rows[1][1].toString()) + rows.each { row -> + assertTrue(row[3] != null, "_row_id should be non-null after MERGE for ${tableName}, row=${row}") + assertTrue(row[4] != null, + "_last_updated_sequence_number should be non-null after MERGE for ${tableName}, row=${row}") + } + assertSparkBusinessRowsEqual(tableName) } formats.each { format -> @@ -167,6 +362,7 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external def dorisTargetTable = tableNameForFormat("v2v3_doris_ops_target", format) log.info("Run v2-to-v3 Doris/Spark compare test with format ${format}") + sql """refresh table ${dbName}.${rowLineageNullTable}""" def scenario1Rows = sql """ select id, _row_id, _last_updated_sequence_number from ${rowLineageNullTable} @@ -175,9 +371,9 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external assertEquals(3, scenario1Rows.size()) scenario1Rows.each { row -> assertTrue(row[1] == null, - "_row_id should be null for rows written before v3 upgrade, row=${row}") + "_row_id should be null for rows written before v3 upgrade, row=${row}") assertTrue(row[2] == null, - "_last_updated_sequence_number should be null for rows written before v3 upgrade, row=${row}") + "_last_updated_sequence_number should be null for rows written before v3 upgrade, row=${row}") } sql """ @@ -199,8 +395,11 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external ) """) assertTrue(dorisRewriteResult.size() > 0, - "Doris rewrite_data_files should return summary rows") + "Doris rewrite_data_files should return summary rows") + assertLineageState(dorisTargetTable, [1, 2, 3, 4], []) + sql """refresh table ${dbName}.${dorisTargetTable}""" + sql """refresh table ${dbName}.${sparkReferenceTable}""" check_sqls_result_equal """ select * from ${dorisTargetTable} @@ -210,11 +409,19 @@ suite("test_iceberg_v2_to_v3_doris_spark_compare", "p0,external,iceberg,external from ${sparkReferenceTable} order by id """ + assertSparkBusinessRowsEqual(dorisTargetTable) upgradeV3DorisOperationInsert(tableNameForFormat("v2v3_doris_upd_case1", format)) upgradeV3DorisOperationDelete(tableNameForFormat("v2v3_doris_upd_case2", format)) upgradeV3DorisOperationUpdate(tableNameForFormat("v2v3_doris_upd_case3", format)) upgradeV3DorisOperationRewrite(tableNameForFormat("v2v3_doris_upd_case4", format)) + upgradeV3DorisOperationMerge(tableNameForFormat("v2v3_doris_upd_case5", format)) + + upgradeV3DorisOperationInsert(tableNameForFormat("v2v3_doris_unpart_case1", format)) + upgradeV3DorisOperationDelete(tableNameForFormat("v2v3_doris_unpart_case2", format)) + upgradeV3DorisOperationUpdate(tableNameForFormat("v2v3_doris_unpart_case3", format)) + validateV3UpgradeReadCompatibility(tableNameForFormat("v2v3_doris_unpart_case4", format)) + upgradeV3DorisOperationMerge(tableNameForFormat("v2v3_doris_unpart_case5", format)) } } finally { diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_doris_create_update_spark_read.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_doris_create_update_spark_read.groovy new file mode 100644 index 00000000000000..67aaa0ff831bd7 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_doris_create_update_spark_read.groovy @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_doris_create_update_spark_read", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_doris_create_update_spark_read" + String dbName = "test_v3_doris_create_update_spark_read_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + def formats = ["parquet", "orc"] + def partitionFlags = [true, false] + + def checksum = { rows -> + assertEquals(1, rows.size()) + return rows[0].collect { col -> col == null ? null : col.toString() } + } + + def assertDorisSparkChecksumEqual = { tableName -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkChecksum = checksum(spark_iceberg(""" + select count(*), sum(id), sum(score), + max(case when id = 2 then tag end) + from demo.${dbName}.${tableName} + """)) + def dorisChecksum = checksum(sql """ + select count(*), sum(id), sum(score), + max(case when id = 2 then tag end) + from ${tableName} + """) + log.info("Doris checksum for ${tableName}: ${dorisChecksum}") + log.info("Spark checksum for ${tableName}: ${sparkChecksum}") + assertEquals(dorisChecksum, sparkChecksum) + } + + def assertDorisSparkBusinessRowsEqual = { tableName -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select id, tag, score, dt + from demo.${dbName}.${tableName} + order by id + """) + def dorisRows = sql(""" + select id, tag, score, dt + from ${tableName} + order by id + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertDorisLineageReadable = { tableName -> + def rows = sql """ + select id, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """ + log.info("Doris lineage rows for ${tableName}: ${rows}") + assertEquals(3, rows.size()) + rows.each { row -> + assertTrue(row[1] != null, "_row_id should be non-null for ${tableName}, row=${row}") + assertTrue(row[2] != null, + "_last_updated_sequence_number should be non-null for ${tableName}, row=${row}") + } + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + + boolean sparkReady = false + try { + spark_iceberg """select 1""" + sparkReady = true + formats.each { format -> + partitionFlags.each { partitioned -> + String partitionSuffix = partitioned ? "part" : "unpart" + String tableName = "doris_v3_update_spark_read_${format}_${partitionSuffix}" + String partitionClause = partitioned ? "partition by list (day(dt)) ()" : "" + + sql """drop table if exists ${tableName}""" + sql """ + create table ${tableName} ( + id int, + tag string, + score int, + dt date + ) engine=iceberg + ${partitionClause} + properties ( + "format-version" = "3", + "write.format.default" = "${format}", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """ + insert into ${tableName} values + (1, 'doris_v3_i', 100, date '2024-03-01'), + (2, 'doris_v3_i', 200, date '2024-03-02'), + (3, 'doris_v3_i', 300, date '2024-03-03') + """ + sql """ + update ${tableName} + set tag = 'doris_v3_u', score = score + 20 + where id = 2 + """ + + def rows = sql """ + select id, tag, score + from ${tableName} + order by id + """ + assertEquals(3, rows.size()) + assertEquals("doris_v3_u", rows[1][1].toString()) + assertEquals(220, rows[1][2].toString().toInteger()) + assertDorisSparkBusinessRowsEqual(tableName) + assertDorisLineageReadable(tableName) + assertDorisSparkChecksumEqual(tableName) + } + } + } catch (Exception e) { + if (!sparkReady) { + logger.info("spark-iceberg JDBC is unavailable, skip Doris v3 create/update Spark read test: ${e.message}") + return + } + throw e + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_complex_query.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_complex_query.groovy new file mode 100644 index 00000000000000..95de1c4ea1a464 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_complex_query.groovy @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_row_lineage_complex_query", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_row_lineage_complex_query" + String dbName = "test_row_lineage_complex_query_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + + def formats = ["parquet", "orc"] + def partitionFlags = [false, true] + + def normalizeRows = { rows -> + return rows.collect { row -> row.collect { col -> col == null ? null : col.toString() } } + } + + def assertSparkDorisRows = { tableName, columns, orderBy, expectedRows = null -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select ${columns} + from demo.${dbName}.${tableName} + order by ${orderBy} + """) + def dorisRows = sql(""" + select ${columns} + from ${tableName} + order by ${orderBy} + """) + log.info("Spark rows for ${tableName}: ${sparkRows}") + log.info("Doris rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + + def normalizedRows = normalizeRows(dorisRows) + if (expectedRows != null) { + assertEquals(expectedRows, normalizedRows) + } + return normalizedRows + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + try { + formats.each { format -> + partitionFlags.each { partitioned -> + String partitionSuffix = partitioned ? "part" : "unpart" + String tableName = "complex_${format}_${partitionSuffix}" + String dimTable = "${tableName}_lineage_dim" + String partitionClause = partitioned ? "partition by list (day(dt)) ()" : "" + try { + sql """drop table if exists ${dimTable}""" + sql """drop table if exists ${tableName}""" + sql """ + create table ${tableName} ( + id int, + name string, + score int, + dt date + ) engine=iceberg + ${partitionClause} + properties ( + "format-version" = "3", + "write.format.default" = "${format}" + ) + """ + + sql """ + insert into ${tableName} values + (1, 'Alice', 10, date '2024-01-01'), + (2, 'Bob', 20, date '2024-01-01'), + (3, 'Carol', 30, date '2024-01-02'), + (4, 'Doris', 40, date '2024-01-02'), + (5, 'Eve', 50, date '2024-01-03') + """ + assertSparkDorisRows(tableName, "id, name, score, dt", "id", [ + ["1", "Alice", "10", "2024-01-01"], + ["2", "Bob", "20", "2024-01-01"], + ["3", "Carol", "30", "2024-01-02"], + ["4", "Doris", "40", "2024-01-02"], + ["5", "Eve", "50", "2024-01-03"] + ]) + + sql """ + create table ${dimTable} ( + id int, + rid bigint, + seq bigint, + tag string + ) engine=iceberg + properties ( + "format-version" = "3", + "write.format.default" = "${format}" + ) + """ + + sql """ + insert into ${dimTable}(id, rid, seq, tag) + select id, _row_id, _last_updated_sequence_number, concat('tag_', cast(id as string)) + from ${tableName} + where id in (1, 4, 5) + """ + + def dimRows = assertSparkDorisRows(dimTable, "id, rid, seq, tag", "id") + log.info("Checking INSERT SELECT into ordinary lineage columns for ${dimTable}: ${dimRows}") + assertEquals(3, dimRows.size()) + dimRows.each { row -> + assertTrue(row[1] != null, "rid should be populated from _row_id for ${dimTable}, row=${row}") + assertTrue(row[2] != null, + "seq should be populated from _last_updated_sequence_number for ${dimTable}, row=${row}") + } + + def joinRows = sql(""" + select t.id, t.name, d.tag + from ${tableName} t + join ${dimTable} d + on t._row_id = d.rid + order by t.id + """) + assertEquals([ + ["1", "Alice", "tag_1"], + ["4", "Doris", "tag_4"], + ["5", "Eve", "tag_5"] + ], normalizeRows(joinRows)) + + def aggregateRows = sql(""" + select dt, count(*), min(_row_id), max(_last_updated_sequence_number) + from ${tableName} + group by dt + order by dt + """) + log.info("Checking row lineage aggregate for ${tableName}: ${aggregateRows}") + assertEquals(3, aggregateRows.size()) + aggregateRows.each { row -> + assertTrue(row[2] != null, "min(_row_id) should be non-null for ${tableName}, row=${row}") + assertTrue(row[3] != null, + "max(_last_updated_sequence_number) should be non-null for ${tableName}, row=${row}") + } + + def subqueryRows = sql(""" + select id, name + from ${tableName} + where _row_id in ( + select rid from ${dimTable} where tag like 'tag_%' + ) + order by id + """) + assertEquals([ + ["1", "Alice"], + ["4", "Doris"], + ["5", "Eve"] + ], normalizeRows(subqueryRows)) + + test { + sql """ + insert into ${tableName}(_row_id, id, name, score, dt) + select rid, id, tag, 0, date '2024-01-01' + from ${dimTable} + """ + exception "Cannot specify row lineage column '_row_id' in INSERT statement" + } + + test { + sql """ + insert into ${tableName}(_last_updated_sequence_number, id, name, score, dt) + select seq, id, tag, 0, date '2024-01-01' + from ${dimTable} + """ + exception "Cannot specify row lineage column '_last_updated_sequence_number' in INSERT statement" + } + } finally { + sql """drop table if exists ${dimTable}""" + sql """drop table if exists ${tableName}""" + } + } + } + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_continuous_dml.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_continuous_dml.groovy new file mode 100644 index 00000000000000..c97db7da5b1e86 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_continuous_dml.groovy @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_row_lineage_continuous_dml", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_row_lineage_continuous_dml" + String dbName = "test_row_lineage_continuous_dml_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + + def formats = ["parquet", "orc"] + def partitionFlags = [true, false] + + def lineageMap = { tableName -> + def rows = sql(""" + select id, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + Map> result = [:] + rows.each { row -> + assertTrue(row[1] != null, "_row_id should be non-null for ${tableName}, row=${row}") + assertTrue(row[2] != null, + "_last_updated_sequence_number should be non-null for ${tableName}, row=${row}") + result[row[0].toString().toInteger()] = [row[1].toString().toLong(), row[2].toString().toLong()] + } + return result + } + + def assertDeleteFilesArePuffin = { tableName -> + def deleteFiles = sql(""" + select file_path, lower(file_format), record_count + from ${tableName}\$delete_files + order by file_path + """) + log.info("Checking continuous DML delete files for ${tableName}: ${deleteFiles}") + assertTrue(deleteFiles.size() > 0, "Continuous MOR DML should create delete files for ${tableName}") + deleteFiles.each { row -> + assertTrue(row[0].toString().toLowerCase().endsWith(".puffin"), + "v3 delete file should be Puffin for ${tableName}, row=${row}") + assertEquals("puffin", row[1].toString()) + } + } + + def assertDeleteFilesDescCaptured = { tableName -> + def descRows = sql("""desc ${tableName}\$delete_files""") + log.info("DESC ${tableName}\$delete_files: ${descRows}") + assertTrue(descRows.size() > 0, "delete_files system table schema should be visible for ${tableName}") + } + + def normalizeBusinessRows = { rows -> + return rows.collect { row -> + [row[0].toString().toInteger(), row[1].toString(), row[2].toString().toInteger(), row[3].toString()] + } + } + + def assertSparkDorisBusinessRows = { tableName, expectedRows = null -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select id, name, score, dt + from demo.${dbName}.${tableName} + order by id + """) + def dorisRows = sql(""" + select id, name, score, dt + from ${tableName} + order by id + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + + def normalizedRows = normalizeBusinessRows(dorisRows) + if (expectedRows != null) { + assertEquals(expectedRows, normalizedRows) + } + return normalizedRows + } + + def assertVisibleRowsAfterV2PositionDeleteMerge = { tableName -> + assertSparkDorisBusinessRows(tableName, [ + [3, "c", 30, "2024-06-02"] + ]) + + def resurrectedRows = sql("""select count(*) from ${tableName} where id in (1, 2)""") + assertEquals(0, resurrectedRows[0][0].toString().toInteger()) + } + + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + try { + formats.each { format -> + partitionFlags.each { partitioned -> + String partitionSuffix = partitioned ? "part" : "unpart" + String tableName = "continuous_dml_${format}_${partitionSuffix}" + String partitionClause = partitioned ? "partition by list (day(dt)) ()" : "" + try { + sql """drop table if exists ${tableName}""" + sql """ + create table ${tableName} ( + id int, + name string, + score int, + dt date + ) engine=iceberg + ${partitionClause} + properties ( + "format-version" = "3", + "write.format.default" = "${format}", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """ + insert into ${tableName} values + (1, 'a', 10, date '2024-06-01'), + (2, 'b', 20, date '2024-06-01'), + (3, 'c', 30, date '2024-06-01'), + (4, 'd', 40, date '2024-06-01'), + (5, 'e', 50, date '2024-06-01') + """ + + Map> beforeLineage = lineageMap(tableName) + sql """delete from ${tableName} where id = 1""" + sql """update ${tableName} set score = score + 100 where id = 2""" + Map> afterUpdateLineage = lineageMap(tableName) + assertEquals(beforeLineage[2][0], afterUpdateLineage[2][0]) + assertTrue(afterUpdateLineage[2][1] > beforeLineage[2][1], + "UPDATE should advance sequence for id=2 in ${tableName}") + + sql """ + merge into ${tableName} t + using ( + select 2 as id, 'b_m' as name, 222 as score, date '2024-06-01' as dt, 'U' as flag + union all + select 3, 'c', 30, date '2024-06-01', 'D' + union all + select 6, 'f', 60, date '2024-06-01', 'I' + ) s + on t.id = s.id + when matched and s.flag = 'D' then delete + when matched then update set name = s.name, score = s.score + when not matched then insert (id, name, score, dt) + values (s.id, s.name, s.score, s.dt) + """ + + assertSparkDorisBusinessRows(tableName, [ + [2, "b_m", 222, "2024-06-01"], + [4, "d", 40, "2024-06-01"], + [5, "e", 50, "2024-06-01"], + [6, "f", 60, "2024-06-01"] + ]) + + Map> afterMergeLineage = lineageMap(tableName) + assertEquals(afterUpdateLineage[2][0], afterMergeLineage[2][0]) + assertTrue(afterMergeLineage[2][1] > afterUpdateLineage[2][1], + "MERGE UPDATE should advance sequence for id=2 in ${tableName}") + assertEquals(beforeLineage[4], afterMergeLineage[4]) + assertEquals(beforeLineage[5], afterMergeLineage[5]) + assertTrue(!afterMergeLineage.containsKey(1), "id=1 should remain deleted in ${tableName}") + assertTrue(!afterMergeLineage.containsKey(3), "id=3 should be deleted by MERGE in ${tableName}") + assertTrue(afterMergeLineage[6][0] != null) + + assertDeleteFilesDescCaptured(tableName) + assertDeleteFilesArePuffin(tableName) + + def snapshots = sql("""select snapshot_id, operation from ${tableName}\$snapshots order by committed_at""") + log.info("Continuous DML snapshots for ${tableName}: ${snapshots}") + assertTrue(snapshots.size() >= 4, "Continuous DML should create snapshots for ${tableName}") + } finally { + sql """drop table if exists ${tableName}""" + } + } + + String v2PositionDeleteTable = "v2_pos_to_v3_dv_${format}" + try { + spark_iceberg_multi """ + drop table if exists demo.${dbName}.${v2PositionDeleteTable}; + create table demo.${dbName}.${v2PositionDeleteTable} ( + id int, + name string, + score int, + dt date + ) using iceberg + partitioned by (days(dt)) + tblproperties ( + 'format-version' = '2', + 'write.format.default' = '${format}', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ); + insert into demo.${dbName}.${v2PositionDeleteTable} values + (1, 'a', 10, date '2024-06-02'), + (2, 'b', 20, date '2024-06-02'), + (3, 'c', 30, date '2024-06-02'); + delete from demo.${dbName}.${v2PositionDeleteTable} where id = 1; + alter table demo.${dbName}.${v2PositionDeleteTable} + set tblproperties ('format-version' = '3'); + """ + + sql """refresh table ${dbName}.${v2PositionDeleteTable}""" + assertSparkDorisBusinessRows(v2PositionDeleteTable, [ + [2, "b", 20, "2024-06-02"], + [3, "c", 30, "2024-06-02"] + ]) + + sql """delete from ${v2PositionDeleteTable} where id = 2""" + assertVisibleRowsAfterV2PositionDeleteMerge(v2PositionDeleteTable) + assertDeleteFilesDescCaptured(v2PositionDeleteTable) + + def deleteFiles = sql(""" + select file_path, lower(file_format), record_count + from ${v2PositionDeleteTable}\$delete_files + order by file_path + """) + log.info("Delete files after v2 position delete to v3 DV merge for ${v2PositionDeleteTable}: ${deleteFiles}") + assertTrue(deleteFiles.any { row -> row[1].toString() == "puffin" }, + "Doris v3 DELETE should create a Puffin DV for ${v2PositionDeleteTable}") + deleteFiles.each { row -> + assertEquals("puffin", row[1].toString()) + assertTrue(row[0].toString().toLowerCase().endsWith(".puffin"), + "live delete files should be rewritten to Puffin DV after v2 position delete merge: ${row}") + } + } finally { + sql """drop table if exists ${v2PositionDeleteTable}""" + } + } + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_dml_mode_matrix.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_dml_mode_matrix.groovy new file mode 100644 index 00000000000000..fede01f4fb441c --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_dml_mode_matrix.groovy @@ -0,0 +1,279 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_row_lineage_dml_mode_matrix", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_row_lineage_dml_mode_matrix" + String dbName = "test_row_lineage_dml_mode_matrix_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + + def formats = ["parquet", "orc"] + def formatVersions = [2, 3] + def partitionFlags = [false, true] + def scenarios = [ + [name: "m01_delete_cow", operation: "delete", property: "write.delete.mode", mode: "copy-on-write"], + [name: "m02_delete_mor", operation: "delete", property: "write.delete.mode", mode: "merge-on-read"], + [name: "m03_update_cow", operation: "update", property: "write.update.mode", mode: "copy-on-write"], + [name: "m04_update_mor", operation: "update", property: "write.update.mode", mode: "merge-on-read"], + [name: "m05_merge_cow", operation: "merge", property: "write.merge.mode", mode: "copy-on-write"], + [name: "m06_merge_mor", operation: "merge", property: "write.merge.mode", mode: "merge-on-read"] + ] + def initialRows = [ + [1, "Alice", 10, "2024-01-01"], + [2, "Bob", 20, "2024-01-02"], + [3, "Carol", 30, "2024-01-03"] + ] + + def createTable = { tableName, formatVersion, format, partitioned, propertyName, mode -> + sql """drop table if exists ${tableName}""" + String partitionClause = partitioned ? "partition by list (day(dt)) ()" : "" + sql """ + create table ${tableName} ( + id int, + name string, + score int, + dt date + ) engine=iceberg + ${partitionClause} + properties ( + "format-version" = "${formatVersion}", + "write.format.default" = "${format}", + "${propertyName}" = "${mode}" + ) + """ + sql """ + insert into ${tableName} values + (1, 'Alice', 10, date '2024-01-01'), + (2, 'Bob', 20, date '2024-01-02'), + (3, 'Carol', 30, date '2024-01-03') + """ + } + + def lineageMap = { tableName -> + def rows = sql(""" + select id, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + Map> result = [:] + rows.each { row -> + result[row[0].toString().toInteger()] = [row[1].toString().toLong(), row[2].toString().toLong()] + } + log.info("Lineage map for ${tableName}: ${result}") + return result + } + + def assertV3LineageNonNull = { tableName, expectedIds -> + def rows = sql(""" + select id, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + log.info("Checking v3 row lineage rows for ${tableName}: ${rows}") + assertEquals(expectedIds.size(), rows.size()) + for (int i = 0; i < expectedIds.size(); i++) { + assertEquals(expectedIds[i], rows[i][0].toString().toInteger()) + assertTrue(rows[i][1] != null, "_row_id should be non-null for ${tableName}, row=${rows[i]}") + assertTrue(rows[i][2] != null, + "_last_updated_sequence_number should be non-null for ${tableName}, row=${rows[i]}") + } + } + + def assertDeleteFilesForVersion = { tableName, formatVersion, format, scenario -> + def deleteFiles = sql(""" + select file_path, lower(file_format), record_count + from ${tableName}\$delete_files + order by file_path + """) + log.info("Checking delete files for ${tableName}, scenario=${scenario.name}: ${deleteFiles}") + if (scenario.mode == "copy-on-write") { + assertEquals(0, deleteFiles.size(), + "COW scenario ${scenario.name} should not create delete files for ${tableName}, actual=${deleteFiles}") + } + if (scenario.mode == "merge-on-read" && formatVersion == 3) { + assertTrue(deleteFiles.size() > 0, + "MOR scenario ${scenario.name} should create delete files for ${tableName}") + } + deleteFiles.each { row -> + String filePath = row[0].toString().toLowerCase() + String fileFormat = row[1].toString().toLowerCase() + if (formatVersion == 3) { + assertEquals("puffin", fileFormat) + assertTrue(filePath.endsWith(".puffin"), + "v3 delete file should be Puffin for ${tableName}, row=${row}") + } else { + assertEquals(format, fileFormat) + assertTrue(filePath.contains("delete_pos"), + "v2 delete file should be position delete for ${tableName}, row=${row}") + assertTrue(filePath.endsWith(format == "parquet" ? ".parquet" : ".orc"), + "v2 delete file suffix should match ${format} for ${tableName}, row=${row}") + } + } + } + + def assertBusinessRows = { tableName, expected -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select id, name, score, dt + from demo.${dbName}.${tableName} + order by id + """) + def dorisRows = sql("""select id, name, score, dt from ${tableName} order by id""") + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + + def actual = dorisRows.collect { row -> + [row[0].toString().toInteger(), row[1].toString(), row[2].toString().toInteger(), row[3].toString()] + } + assertEquals(expected, actual) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + try { + scenarios.each { scenario -> + formatVersions.each { formatVersion -> + formats.each { format -> + partitionFlags.each { partitioned -> + String partitionSuffix = partitioned ? "part" : "unpart" + String tableName = "${scenario.name}_v${formatVersion}_${format}_${partitionSuffix}" + log.info("Running DML mode scenario ${scenario} on ${tableName}") + try { + createTable(tableName, formatVersion, format, partitioned, scenario.property, scenario.mode) + Map> beforeLineage = formatVersion == 3 ? lineageMap(tableName) : [:] + + String dmlSql + def expectedRows + def expectedIds + if (scenario.operation == "delete") { + dmlSql = """delete from ${tableName} where id = 2""" + expectedRows = [ + [1, "Alice", 10, "2024-01-01"], + [3, "Carol", 30, "2024-01-03"] + ] + expectedIds = [1, 3] + } else if (scenario.operation == "update") { + dmlSql = """update ${tableName} set name = 'Alice_u', score = score + 100 where id = 1""" + expectedRows = [ + [1, "Alice_u", 110, "2024-01-01"], + [2, "Bob", 20, "2024-01-02"], + [3, "Carol", 30, "2024-01-03"] + ] + expectedIds = [1, 2, 3] + } else { + dmlSql = """ + merge into ${tableName} t + using ( + select 1 as id, 'Alice_m' as name, 111 as score, date '2024-01-01' as dt, 'U' as flag + union all + select 3, 'Carol', 30, date '2024-01-03', 'D' + union all + select 5, 'Eve', 50, date '2024-01-05', 'I' + ) s + on t.id = s.id + when matched and s.flag = 'D' then delete + when matched then update set name = s.name, score = s.score + when not matched then insert (id, name, score, dt) + values (s.id, s.name, s.score, s.dt) + """ + expectedRows = [ + [1, "Alice_m", 111, "2024-01-01"], + [2, "Bob", 20, "2024-01-02"], + [5, "Eve", 50, "2024-01-05"] + ] + expectedIds = [1, 2, 5] + } + + if (scenario.mode == "copy-on-write") { + String operationName = scenario.operation == "merge" + ? "MERGE INTO" : scenario.operation.toUpperCase() + test { + sql dmlSql + exception "Doris does not support ${operationName} on Iceberg copy-on-write tables" + exception "Set table property '${scenario.property}' to 'merge-on-read'" + } + assertBusinessRows(tableName, initialRows) + if (formatVersion == 3) { + assertV3LineageNonNull(tableName, [1, 2, 3]) + assertEquals(beforeLineage, lineageMap(tableName)) + } + } else { + sql dmlSql + assertBusinessRows(tableName, expectedRows) + if (formatVersion == 3) { + assertV3LineageNonNull(tableName, expectedIds) + Map> afterLineage = lineageMap(tableName) + if (scenario.operation == "delete") { + assertEquals(beforeLineage[1], afterLineage[1]) + assertEquals(beforeLineage[3], afterLineage[3]) + assertTrue(!afterLineage.containsKey(2)) + } else if (scenario.operation == "update") { + assertEquals(beforeLineage[1][0], afterLineage[1][0]) + assertTrue(afterLineage[1][1] > beforeLineage[1][1], + "UPDATE should advance sequence for id=1 in ${tableName}") + assertEquals(beforeLineage[3], afterLineage[3]) + } else { + assertEquals(beforeLineage[1][0], afterLineage[1][0]) + assertTrue(afterLineage[1][1] > beforeLineage[1][1], + "MERGE UPDATE should advance sequence for id=1 in ${tableName}") + assertEquals(beforeLineage[2], afterLineage[2]) + assertTrue(!afterLineage.containsKey(3)) + assertTrue(afterLineage[5][0] != null) + } + } + } + + assertDeleteFilesForVersion(tableName, formatVersion, format, scenario) + } finally { + sql """drop table if exists ${tableName}""" + } + } + } + } + } + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_equality_delete_dv_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_equality_delete_dv_interop.groovy new file mode 100644 index 00000000000000..8444f80e83a05d --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_equality_delete_dv_interop.groovy @@ -0,0 +1,317 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_row_lineage_equality_delete_dv_interop", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_row_lineage_equality_delete_dv_interop" + String dbName = "test_row_lineage_eq_delete_dv_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + String runSuffix = "${System.currentTimeMillis()}" + + def fixtureMetadataFiles = [ + parquet: "s3a://warehouse/wh/multi_catalog/equality_delete_par_1/metadata/00011-f7268cac-dd2f-4061-a2c9-b5aa54ab52f4.metadata.json", + orc: "s3a://warehouse/wh/multi_catalog/equality_delete_orc_1/metadata/00011-568385a2-14af-4237-b186-ce46d350be19.metadata.json" + ] + + def normalizeRows = { rows -> + return rows.collect { row -> row.collect { col -> col == null ? null : col.toString() } } + } + + def unregisterSparkTable = { tableName -> + try { + spark_iceberg """ + call demo.system.unregister_table(table => '${dbName}.${tableName}') + """ + } catch (Throwable t) { + logger.info("Skip unregister ${dbName}.${tableName}: ${t.message}") + } + } + + def refreshTable = { tableName -> + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + sql """refresh table ${dbName}.${tableName}""" + } + + def assertSparkDorisBusinessRows = { tableName -> + refreshTable(tableName) + def sparkRows = spark_iceberg(""" + select new_new_id, new_name, data, id + from demo.${dbName}.${tableName} + order by new_new_id, new_name, data, id + """) + def dorisRows = sql(""" + select new_new_id, new_name, data, id + from ${tableName} + order by new_new_id, new_name, data, id + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertDorisBusinessRows = { tableName, expectedRows -> + refreshTable(tableName) + def sparkRows = spark_iceberg(""" + select new_new_id, new_name, data, id + from demo.${dbName}.${tableName} + order by new_new_id, new_name, data, id + """) + def dorisRows = sql(""" + select new_new_id, new_name, data, id + from ${tableName} + order by new_new_id, new_name, data, id + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + def normalizedRows = normalizeRows(dorisRows) + log.info("Checking concrete Doris business rows for ${tableName}: ${normalizedRows}") + assertEquals(expectedRows, normalizedRows) + } + + def assertSparkDorisAggregateRows = { tableName -> + refreshTable(tableName) + def sparkRows = spark_iceberg(""" + select count(*), count(distinct new_new_id) + from demo.${dbName}.${tableName} + """) + def dorisRows = sql(""" + select count(*), count(distinct new_new_id) + from ${tableName} + """) + log.info("Spark aggregate rows for ${tableName}: ${sparkRows}") + log.info("Doris aggregate rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertDorisAggregateRows = { tableName, expectedRows -> + refreshTable(tableName) + def sparkRows = spark_iceberg(""" + select count(*), count(distinct new_new_id) + from demo.${dbName}.${tableName} + """) + def dorisRows = sql(""" + select count(*), count(distinct new_new_id) + from ${tableName} + """) + log.info("Spark aggregate rows for ${tableName}: ${sparkRows}") + log.info("Doris aggregate rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + def normalizedRows = normalizeRows(dorisRows) + log.info("Checking concrete Doris aggregate rows for ${tableName}: ${normalizedRows}") + assertEquals(expectedRows, normalizedRows) + } + + def assertSparkDorisRowsByPredicate = { tableName, predicate -> + refreshTable(tableName) + def sparkRows = spark_iceberg(""" + select new_new_id, new_name, data, id + from demo.${dbName}.${tableName} + where ${predicate} + order by new_new_id, new_name, data, id + """) + def dorisRows = sql(""" + select new_new_id, new_name, data, id + from ${tableName} + where ${predicate} + order by new_new_id, new_name, data, id + """) + log.info("Spark rows for ${tableName} with predicate ${predicate}: ${sparkRows}") + log.info("Doris rows for ${tableName} with predicate ${predicate}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertDorisRowsByPredicate = { tableName, predicate, expectedRows -> + refreshTable(tableName) + def sparkRows = spark_iceberg(""" + select new_new_id, new_name, data, id + from demo.${dbName}.${tableName} + where ${predicate} + order by new_new_id, new_name, data, id + """) + def dorisRows = sql(""" + select new_new_id, new_name, data, id + from ${tableName} + where ${predicate} + order by new_new_id, new_name, data, id + """) + log.info("Spark rows for ${tableName} with predicate ${predicate}: ${sparkRows}") + log.info("Doris rows for ${tableName} with predicate ${predicate}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + def normalizedRows = normalizeRows(dorisRows) + log.info("Checking concrete Doris rows for ${tableName} with predicate ${predicate}: ${normalizedRows}") + assertEquals(expectedRows, normalizedRows) + } + + def assertLineageProjectionReadable = { tableName, expectNonNullLineage -> + def rows = sql(""" + select new_new_id, _row_id, _last_updated_sequence_number + from ${tableName} + order by new_new_id + """) + logger.info("Row lineage projection for ${tableName}, expectNonNullLineage=${expectNonNullLineage}: ${rows}") + assertTrue(rows.size() > 0, "Row lineage projection should return visible rows for ${tableName}") + rows.each { row -> + if (expectNonNullLineage) { + assertTrue(row[1] != null, + "_row_id should be non-null after v2-to-v3 write in ${tableName}, row=${row}") + assertTrue(row[2] != null, + "_last_updated_sequence_number should be non-null after v2-to-v3 write in ${tableName}, row=${row}") + } else { + assertEquals(null, row[1], + "_row_id should be NULL before v2-to-v3 write in ${tableName}, row=${row}") + assertEquals(null, row[2], + "_last_updated_sequence_number should be NULL before v2-to-v3 write in ${tableName}, row=${row}") + } + } + } + + def deleteFileFormats = { tableName -> + refreshTable(tableName) + def sparkRows = spark_iceberg(""" + select lower(file_format), count(*) + from demo.${dbName}.${tableName}.delete_files + group by lower(file_format) + order by lower(file_format) + """) + def dorisRows = sql(""" + select lower(file_format), count(*) + from ${tableName}\$delete_files + group by lower(file_format) + order by lower(file_format) + """) + log.info("Spark delete file formats for ${tableName}: ${sparkRows}") + log.info("Doris delete file formats for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + return dorisRows + } + + def assertOnlyEqualityDeleteFilesBeforeDorisDelete = { tableName -> + def deleteFiles = deleteFileFormats(tableName) + logger.info("Delete files before Doris v3 DELETE for ${tableName}: ${deleteFiles}") + assertTrue(deleteFiles.size() > 0, "Fixture should contain equality delete files for ${tableName}") + assertTrue(!deleteFiles.any { row -> row[0].toString() == "puffin" }, + "Fixture should not contain Puffin DV before Doris DELETE for ${tableName}: ${deleteFiles}") + } + + def assertEqualityDeleteAndPuffinDvCoexist = { tableName -> + def deleteFiles = deleteFileFormats(tableName) + logger.info("Delete files after Doris v3 DELETE for ${tableName}: ${deleteFiles}") + assertTrue(deleteFiles.any { row -> row[0].toString() == "puffin" }, + "Doris v3 DELETE should write Puffin DV for ${tableName}: ${deleteFiles}") + assertTrue(deleteFiles.any { row -> row[0].toString() != "puffin" }, + "Existing equality delete files should still be visible with Puffin DV for ${tableName}: ${deleteFiles}") + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + String oldVersionPredicate = """ + (new_new_id = 1 and new_name in ('smith', 'smith2', 'smith3')) + or (new_new_id = 2 and new_name = 'danny') + or (new_new_id = 4 and new_name in ('bob', 'bob2')) + or (new_new_id = 5 and new_name = 'dennis') + """ + + def expectedRowsAfterDorisDelete = [ + parquet: [ + ["1", "smith4", "aaaa", "1"], + ["3", "alice", "c", null], + ["4", "bob3", "eee", null], + ["5", "dennis2", "ff", null], + ["6", "jasson", "g", null], + ["7", "parker", "h", null] + ], + orc: [ + ["1", "smith4", "aaaa", "1"], + ["3", "alice", "c", null], + ["4", "bob3", "eee", null], + ["5", "dennis2", "ff", null], + ["6", "jasson", "g", null], + ["7", "orcker", "h", null] + ] + ] + + try { + spark_iceberg """create database if not exists demo.${dbName}""" + + fixtureMetadataFiles.each { format, metadataFile -> + String tableName = "eq_delete_dv_${format}_${runSuffix}" + try { + spark_iceberg_multi """ + call demo.system.register_table( + table => '${dbName}.${tableName}', + metadata_file => '${metadataFile}' + ); + alter table demo.${dbName}.${tableName} + set tblproperties ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ); + """ + + assertSparkDorisBusinessRows(tableName) + assertSparkDorisAggregateRows(tableName) + assertSparkDorisRowsByPredicate(tableName, oldVersionPredicate) + assertLineageProjectionReadable(tableName, false) + assertOnlyEqualityDeleteFilesBeforeDorisDelete(tableName) + + sql """ + delete from ${tableName} + where new_new_id = 2 + """ + + assertDorisBusinessRows(tableName, expectedRowsAfterDorisDelete[format]) + assertDorisAggregateRows(tableName, [["6", "6"]]) + assertDorisRowsByPredicate(tableName, "new_new_id = 2", []) + assertSparkDorisRowsByPredicate(tableName, oldVersionPredicate) + assertLineageProjectionReadable(tableName, true) + assertEqualityDeleteAndPuffinDvCoexist(tableName) + } finally { + unregisterSparkTable(tableName) + } + } + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_query_insert.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_query_insert.groovy index 28e28bdd8870b8..1d14631402fcb1 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_query_insert.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_query_insert.groovy @@ -35,6 +35,34 @@ suite("test_iceberg_v3_row_lineage_query_insert", "p0,external,iceberg,external_ return rows.collect { row -> row[0].toString().toLowerCase() } } + def normalizeRows = { rows -> + return rows.collect { row -> row.collect { col -> col == null ? null : col.toString() } } + } + + def assertSparkDorisBusinessRows = { tableName, columns, orderBy, expectedRows = null -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select ${columns} + from demo.${dbName}.${tableName} + order by ${orderBy} + """) + def dorisRows = sql(""" + select ${columns} + from ${tableName} + order by ${orderBy} + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + + def normalizedRows = normalizeRows(dorisRows) + if (expectedRows != null) { + assertEquals(expectedRows, normalizedRows) + } + return normalizedRows + } + def schemaContainsField = { schemaRows, fieldName -> String target = fieldName.toLowerCase() return schemaRows.any { row -> row.toString().toLowerCase().contains(target) } @@ -106,7 +134,7 @@ suite("test_iceberg_v3_row_lineage_query_insert", "p0,external,iceberg,external_ def rowLineageRows = sql(""" select id, _row_id, _last_updated_sequence_number from ${tableName} - order by id + order by id, _row_id """) log.info("Checking explicit row lineage projection for ${tableName}: rows=${rowLineageRows}") assertEquals(expectedIds.size(), rowLineageRows.size()) @@ -138,6 +166,111 @@ suite("test_iceberg_v3_row_lineage_query_insert", "p0,external,iceberg,external_ assertEquals(2, combinedPredicate.size()) assertEquals(expectedIds[1], combinedPredicate[0][0].toString().toInteger()) assertEquals(expectedIds[2], combinedPredicate[1][0].toString().toInteger()) + + def aggregateRows = sql(""" + select count(*), min(_row_id), max(_last_updated_sequence_number) + from ${tableName} + where _row_id is not null + """) + log.info("Checking row lineage aggregate for ${tableName}: result=${aggregateRows}") + assertEquals(expectedIds.size(), aggregateRows[0][0].toString().toInteger()) + assertTrue(aggregateRows[0][1] != null, "min(_row_id) should be non-null for ${tableName}") + assertTrue(aggregateRows[0][2] != null, + "max(_last_updated_sequence_number) should be non-null for ${tableName}") + } + + def assertV2DoesNotExposeRowLineageColumns = { tableName -> + sql("""set show_hidden_columns = true""") + def descRows = sql("""desc ${tableName}""") + def columns = collectDescColumns(descRows) + log.info("Checking v2 row lineage compatibility for ${tableName}: desc=${descRows}") + assertTrue(!columns.contains("_row_id"), + "Iceberg v2 table should not expose _row_id for ${tableName}, got ${columns}") + assertTrue(!columns.contains("_last_updated_sequence_number"), + "Iceberg v2 table should not expose _last_updated_sequence_number for ${tableName}, got ${columns}") + + test { + sql """select id, _row_id from ${tableName}""" + exception "_row_id" + } + + test { + sql """select id, _last_updated_sequence_number from ${tableName}""" + exception "_last_updated_sequence_number" + } + + sql("""set show_hidden_columns = false""") + } + + def assertRowLineagePredicatesAreNotPushedDown = { tableName, rowId, sequenceNumber -> + def explainRowId = sql(""" + explain verbose + select id + from ${tableName} + where id = 1 and _row_id = ${rowId} + """) + def explainSeq = sql(""" + explain verbose + select id + from ${tableName} + where _last_updated_sequence_number >= ${sequenceNumber} + """) + + [explainRowId, explainSeq].each { explainRows -> + String pushdownText = explainRows.collect { row -> row.toString().toLowerCase() } + .findAll { line -> line.contains("icebergpredicatepushdown") } + .join("\n") + log.info("Checking row lineage predicate pushdown for ${tableName}: ${pushdownText}") + assertTrue(!pushdownText.contains("_row_id"), + "_row_id should not be pushed to Iceberg predicate pushdown for ${tableName}: ${pushdownText}") + assertTrue(!pushdownText.contains("_last_updated_sequence_number"), + "_last_updated_sequence_number should not be pushed to Iceberg predicate pushdown for ${tableName}: ${pushdownText}") + } + } + + def assertCreateTableWithRowLineageColumnsFails = { format -> + test { + sql """ + create table create_with_row_id_${format} ( + id int, + _row_id bigint + ) engine=iceberg + properties ( + "format-version" = "3", + "write.format.default" = "${format}" + ) + """ + exception "Cannot create Iceberg v3 table with reserved row lineage column: _row_id" + } + + test { + sql """ + create table create_with_sequence_${format} ( + id int, + _last_updated_sequence_number bigint + ) engine=iceberg + properties ( + "format-version" = "3", + "write.format.default" = "${format}" + ) + """ + exception "Cannot create Iceberg v3 table with reserved row lineage column: _last_updated_sequence_number" + } + + test { + sql """ + create table create_with_two_lineage_cols_${format} ( + id int, + _row_id bigint, + _last_updated_sequence_number bigint + ) engine=iceberg + properties ( + "format-version" = "3", + "write.format.default" = "${format}" + ) + """ + exception "Cannot create Iceberg v3 table with reserved row lineage column: _row_id" + } } def assertRowLineageOnlyAggregatesReadable = { tableName, expectedRowCount -> @@ -213,9 +346,26 @@ suite("test_iceberg_v3_row_lineage_query_insert", "p0,external,iceberg,external_ formats.each { format -> String unpartitionedTable = "test_row_lineage_query_insert_unpartitioned_${format}" String partitionedTable = "test_row_lineage_query_insert_partitioned_${format}" + String v2Table = "test_row_lineage_query_insert_v2_${format}" log.info("Run row lineage query/insert test with format ${format}") try { + assertCreateTableWithRowLineageColumnsFails(format) + + sql """drop table if exists ${v2Table}""" + sql """ + create table ${v2Table} ( + id int, + name string + ) engine=iceberg + properties ( + "format-version" = "2", + "write.format.default" = "${format}" + ) + """ + sql """insert into ${v2Table} values(1, 'legacy')""" + assertV2DoesNotExposeRowLineageColumns(v2Table) + sql """drop table if exists ${unpartitionedTable}""" sql """ create table ${unpartitionedTable} ( @@ -235,17 +385,35 @@ suite("test_iceberg_v3_row_lineage_query_insert", "p0,external,iceberg,external_ (2, 'Bob', 30), (3, 'Charlie', 35) """ + sql """ insert into ${unpartitionedTable} values(2, 'Bob', 30) """ + sql """ insert into ${unpartitionedTable} values(3, 'Charlie', 35) """ log.info("Inserted initial rows into ${unpartitionedTable}") + assertSparkDorisBusinessRows(unpartitionedTable, "id, name, age", "id, name, age", [ + ["1", "Alice", "25"], + ["2", "Bob", "30"], + ["2", "Bob", "30"], + ["3", "Charlie", "35"], + ["3", "Charlie", "35"] + ]) // Assert baseline: // 1. DESC and SELECT * hide row lineage columns by default. // 2. show_hidden_columns=true exposes both hidden columns in DESC and SELECT *. // 3. Explicit SELECT on row lineage columns returns non-null values. assertRowLineageHiddenColumns(unpartitionedTable, 3) - assertExplicitRowLineageReadable(unpartitionedTable, [1, 2, 3]) + assertExplicitRowLineageReadable(unpartitionedTable, [1, 2, 2, 3, 3]) + def unpartitionedLineageRows = sql(""" + select id, _row_id, _last_updated_sequence_number + from ${unpartitionedTable} + order by id, _row_id + """) + assertRowLineagePredicatesAreNotPushedDown( + unpartitionedTable, + unpartitionedLineageRows[0][1], + unpartitionedLineageRows[0][2]) if (format == "parquet") { - assertRowLineageOnlyAggregatesReadable(unpartitionedTable, 3) + assertRowLineageOnlyAggregatesReadable(unpartitionedTable, 5) } test { @@ -264,14 +432,22 @@ suite("test_iceberg_v3_row_lineage_query_insert", "p0,external,iceberg,external_ sql """insert into ${unpartitionedTable}(id, name, age) values (4, 'Doris', 40)""" def unpartitionedCount = sql """select count(*) from ${unpartitionedTable}""" log.info("Checking row count after regular INSERT for ${unpartitionedTable}: result=${unpartitionedCount}") - assertEquals(4, unpartitionedCount[0][0].toString().toInteger()) + assertEquals(6, unpartitionedCount[0][0].toString().toInteger()) + assertSparkDorisBusinessRows(unpartitionedTable, "id, name, age", "id, name, age", [ + ["1", "Alice", "25"], + ["2", "Bob", "30"], + ["2", "Bob", "30"], + ["3", "Charlie", "35"], + ["3", "Charlie", "35"], + ["4", "Doris", "40"] + ]) assertCurrentFilesDoNotContainRowLineageColumns( unpartitionedTable, format, "Unpartitioned normal INSERT") if (format == "parquet") { - assertRowLineageOnlyAggregatesReadable(unpartitionedTable, 4) + assertRowLineageOnlyAggregatesReadable(unpartitionedTable, 6) } sql """drop table if exists ${partitionedTable}""" @@ -294,6 +470,11 @@ suite("test_iceberg_v3_row_lineage_query_insert", "p0,external,iceberg,external_ sql """ insert into ${partitionedTable} values(13, 'Rita', 23, '2024-01-03')""" log.info("Inserted initial rows into ${partitionedTable}") + assertSparkDorisBusinessRows(partitionedTable, "id, name, age, dt", "id", [ + ["11", "Penny", "21", "2024-01-01"], + ["12", "Quinn", "22", "2024-01-02"], + ["13", "Rita", "23", "2024-01-03"] + ]) // Assert baseline: // 1. Partitioned tables follow the same row lineage semantics as unpartitioned tables. @@ -323,6 +504,26 @@ suite("test_iceberg_v3_row_lineage_query_insert", "p0,external,iceberg,external_ assertEquals(1, exactPartitionPredicate.size()) assertEquals(12, exactPartitionPredicate[0][0].toString().toInteger()) + def partitionAggregate = sql(""" + select dt, count(*), min(_row_id), max(_last_updated_sequence_number) + from ${partitionedTable} + group by dt + order by dt + """) + log.info("Checking partitioned row lineage aggregate for ${partitionedTable}: result=${partitionAggregate}") + assertEquals(3, partitionAggregate.size()) + partitionAggregate.each { row -> + assertEquals(1, row[1].toString().toInteger()) + assertTrue(row[2] != null, "partition min(_row_id) should be non-null for ${partitionedTable}") + assertTrue(row[3] != null, + "partition max(_last_updated_sequence_number) should be non-null for ${partitionedTable}") + } + + assertRowLineagePredicatesAreNotPushedDown( + partitionedTable, + partitionLineageRows[1][1], + partitionLineageRows[1][2]) + test { sql """ insert into ${partitionedTable}(_row_id, id, name, age, dt) @@ -343,12 +544,23 @@ suite("test_iceberg_v3_row_lineage_query_insert", "p0,external,iceberg,external_ def partitionedCount = sql """select count(*) from ${partitionedTable}""" log.info("Checking row count after regular INSERT for ${partitionedTable}: result=${partitionedCount}") assertEquals(4, partitionedCount[0][0].toString().toInteger()) + assertSparkDorisBusinessRows(partitionedTable, "id, name, age, dt", "id", [ + ["11", "Penny", "21", "2024-01-01"], + ["12", "Quinn", "22", "2024-01-02"], + ["13", "Rita", "23", "2024-01-03"], + ["14", "Sara", "24", "2024-01-04"] + ]) assertCurrentFilesDoNotContainRowLineageColumns( partitionedTable, format, "Partitioned normal INSERT") } finally { + sql """set show_hidden_columns = false""" + sql """drop table if exists create_with_two_lineage_cols_${format}""" + sql """drop table if exists create_with_sequence_${format}""" + sql """drop table if exists create_with_row_id_${format}""" + sql """drop table if exists ${v2Table}""" sql """drop table if exists ${partitionedTable}""" sql """drop table if exists ${unpartitionedTable}""" } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_schema_evolution.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_schema_evolution.groovy new file mode 100644 index 00000000000000..90e90fd56ee0cb --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_schema_evolution.groovy @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_row_lineage_schema_evolution", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_row_lineage_schema_evolution" + String dbName = "test_row_lineage_schema_evolution_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + + def formats = ["parquet", "orc"] + def partitionFlags = [true, false] + + def descColumns = { tableName -> return sql("""desc ${tableName}""").collect { row -> row[0].toString().toLowerCase() } + } + + def normalizeRows = { rows -> + return rows.collect { row -> row.collect { col -> col == null ? null : col.toString() } } + } + + def assertSparkDorisBusinessRows = { tableName, columns, orderBy, expectedRows = null -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select ${columns} + from demo.${dbName}.${tableName} + order by ${orderBy} + """) + def dorisRows = sql(""" + select ${columns} + from ${tableName} + order by ${orderBy} + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + + def normalizedRows = normalizeRows(dorisRows) + if (expectedRows != null) { + assertEquals(expectedRows, normalizedRows) + } + return normalizedRows + } + + def lineageMap = { tableName -> + def rows = sql(""" + select id, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + Map> result = [:] + rows.each { row -> + assertTrue(row[1] != null, "_row_id should be non-null for ${tableName}, row=${row}") + assertTrue(row[2] != null, + "_last_updated_sequence_number should be non-null for ${tableName}, row=${row}") + result[row[0].toString().toInteger()] = [row[1].toString().toLong(), row[2].toString().toLong()] + } + return result + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + try { + formats.each { format -> + partitionFlags.each { partitioned -> + String partitionSuffix = partitioned ? "part" : "unpart" + String tableName = "schema_evo_${format}_${partitionSuffix}" + String partitionClause = partitioned ? "partition by list (day(dt)) ()" : "" + try { + sql """drop table if exists ${tableName}""" + sql """ + create table ${tableName} ( + id int, + name string, + score int, + dt date + ) engine=iceberg + ${partitionClause} + properties ( + "format-version" = "3", + "write.format.default" = "${format}", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """ + insert into ${tableName} values + (1, 'a', 10, date '2024-10-01'), + (2, 'b', 20, date '2024-10-01') + """ + + Map> beforeLineage = lineageMap(tableName) + def initialColumns = descColumns(tableName) + assertTrue(!initialColumns.contains("_row_id")) + assertTrue(!initialColumns.contains("_last_updated_sequence_number")) + + sql """alter table ${tableName} add column extra string""" + sql """refresh table ${dbName}.${tableName}""" + def addColumns = descColumns(tableName) + assertTrue(addColumns.contains("extra"), "ADD COLUMN should expose extra for ${tableName}") + + def afterAddRows = sql(""" + select id, name, score, extra, dt, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + log.info("Rows after ADD COLUMN for ${tableName}: ${afterAddRows}") + assertEquals(2, afterAddRows.size()) + afterAddRows.each { row -> + assertTrue(row[3] == null, "New extra column should be null for old rows in ${tableName}") + assertTrue(row[5] != null, "_row_id should remain readable after ADD COLUMN in ${tableName}") + assertTrue(row[6] != null, + "_last_updated_sequence_number should remain readable after ADD COLUMN in ${tableName}") + } + assertSparkDorisBusinessRows(tableName, "id, name, score, extra, dt", "id", [ + ["1", "a", "10", null, "2024-10-01"], + ["2", "b", "20", null, "2024-10-01"] + ]) + + sql """alter table ${tableName} drop column extra""" + sql """refresh table ${dbName}.${tableName}""" + def dropColumns = descColumns(tableName) + assertTrue(!dropColumns.contains("extra"), "DROP COLUMN should remove extra for ${tableName}") + + def afterDropRows = sql(""" + select id, name, score, dt, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + log.info("Rows after DROP COLUMN for ${tableName}: ${afterDropRows}") + assertEquals(2, afterDropRows.size()) + assertEquals("a", afterDropRows[0][1].toString()) + assertEquals(10, afterDropRows[0][2].toString().toInteger()) + assertEquals("2024-10-01", afterDropRows[0][3].toString()) + assertSparkDorisBusinessRows(tableName, "id, name, score, dt", "id", [ + ["1", "a", "10", "2024-10-01"], + ["2", "b", "20", "2024-10-01"] + ]) + + sql """update ${tableName} set score = score + 100 where id = 1""" + Map> afterUpdateLineage = lineageMap(tableName) + assertEquals(beforeLineage[1][0], afterUpdateLineage[1][0]) + assertTrue(afterUpdateLineage[1][1] > beforeLineage[1][1], + "UPDATE after schema evolution should advance sequence for id=1 in ${tableName}") + assertEquals(beforeLineage[2], afterUpdateLineage[2]) + + def finalRows = sql(""" + select id, name, score, dt, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + assertEquals(110, finalRows[0][2].toString().toInteger()) + assertEquals(20, finalRows[1][2].toString().toInteger()) + assertSparkDorisBusinessRows(tableName, "id, name, score, dt", "id", [ + ["1", "a", "110", "2024-10-01"], + ["2", "b", "20", "2024-10-01"] + ]) + + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + spark_iceberg """ + alter table demo.${dbName}.${tableName} + add columns (spark_added string) + """ + sql """refresh table ${dbName}.${tableName}""" + def sparkAddColumns = descColumns(tableName) + assertTrue(sparkAddColumns.contains("spark_added"), + "Spark ADD COLUMN should be visible after Doris REFRESH TABLE for ${tableName}") + + def afterSparkAddRows = sql(""" + select id, name, score, spark_added, dt, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + log.info("Rows after Spark ADD COLUMN for ${tableName}: ${afterSparkAddRows}") + assertEquals(2, afterSparkAddRows.size()) + afterSparkAddRows.each { row -> + assertTrue(row[3] == null, + "Spark-added column should be null for existing rows in ${tableName}") + assertTrue(row[5] != null, "_row_id should remain readable after Spark schema evolution") + assertTrue(row[6] != null, + "_last_updated_sequence_number should remain readable after Spark schema evolution") + } + assertSparkDorisBusinessRows(tableName, "id, name, score, spark_added, dt", "id", [ + ["1", "a", "110", null, "2024-10-01"], + ["2", "b", "20", null, "2024-10-01"] + ]) + } finally { + sql """drop table if exists ${tableName}""" + } + } + } + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_spark_doris_dv_interop.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_spark_doris_dv_interop.groovy new file mode 100644 index 00000000000000..bd64715ecab848 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_spark_doris_dv_interop.groovy @@ -0,0 +1,294 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_row_lineage_spark_doris_dv_interop", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_row_lineage_spark_doris_dv_interop" + String dbName = "test_row_lineage_spark_doris_dv_interop_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + + def formats = ["parquet", "orc"] + def partitionFlags = [true, false] + + def normalizeRows = { rows -> + return rows.collect { row -> row.collect { col -> col == null ? null : col.toString() } } + } + + def assertSparkDorisBusinessRows = { tableName -> + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + sql """refresh table ${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select id, name, score, dt + from demo.${dbName}.${tableName} + order by id + """) + def dorisRows = sql(""" + select id, name, score, dt + from ${tableName} + order by id + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertDorisBusinessRows = { tableName, expectedRows -> + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + sql """refresh table ${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select id, name, score, dt + from demo.${dbName}.${tableName} + order by id + """) + def dorisRows = sql(""" + select id, name, score, dt + from ${tableName} + order by id + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + def normalizedRows = normalizeRows(dorisRows) + log.info("Checking concrete Doris business rows for ${tableName}: ${normalizedRows}") + assertEquals(expectedRows, normalizedRows) + } + + def assertSparkDorisAggregateRows = { tableName -> + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + sql """refresh table ${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select count(*), sum(id), sum(score) + from demo.${dbName}.${tableName} + """) + def dorisRows = sql(""" + select count(*), sum(id), sum(score) + from ${tableName} + """) + log.info("Spark aggregate rows for ${tableName}: ${sparkRows}") + log.info("Doris aggregate rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertDorisAggregateRows = { tableName, expectedRows -> + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + sql """refresh table ${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select count(*), sum(id), sum(score) + from demo.${dbName}.${tableName} + """) + def dorisRows = sql(""" + select count(*), sum(id), sum(score) + from ${tableName} + """) + log.info("Spark aggregate rows for ${tableName}: ${sparkRows}") + log.info("Doris aggregate rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + def normalizedRows = normalizeRows(dorisRows) + log.info("Checking concrete Doris aggregate rows for ${tableName}: ${normalizedRows}") + assertEquals(expectedRows, normalizedRows) + } + + def assertDorisLineageReadable = { tableName -> + def rows = sql(""" + select id, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + log.info("Checking row lineage in ${tableName}: ${rows}") + rows.each { row -> + assertTrue(row[1] != null, "_row_id should be non-null for ${tableName}, row=${row}") + assertTrue(row[2] != null, + "_last_updated_sequence_number should be non-null for ${tableName}, row=${row}") + } + } + + def deleteFiles = { tableName -> + return sql(""" + select file_path, lower(file_format) + from ${tableName}\$delete_files + order by file_path + """) + } + + def assertPuffinDeleteFiles = { tableName -> + def rows = deleteFiles(tableName) + + log.info("Checking Puffin delete files" + + " for ${tableName}: ${rows}") + assertTrue(rows.size() > 0, "v3 delete should create delete files for ${tableName}") + rows.each { row -> + assertEquals("puffin", row[1].toString()) + assertTrue(row[0].toString().toLowerCase().endsWith(".puffin")) + } + } + + def assertSparkGeneratedDvReadable = { + spark_iceberg """refresh table demo.format_v3.dv_test""" + sql """refresh table format_v3.dv_test""" + assertPuffinDeleteFiles("format_v3.dv_test") + def sparkRows = spark_iceberg(""" + select id + from demo.format_v3.dv_test + order by id + """) + def dorisRows = sql(""" + select id + from format_v3.dv_test + order by id + """) + log.info("Checking Spark/Doris read result for Spark-generated DV fixture format_v3.dv_test: " + + "Spark=${sparkRows}, Doris=${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + + def rows = sql("""select id, _row_id, _last_updated_sequence_number from format_v3.dv_test order by id""") + log.info("Checking Doris row lineage for Spark-generated DV fixture format_v3.dv_test: ${rows}") + rows.each { row -> + assertTrue(row[1] != null, "_row_id should be non-null for Spark DV fixture, row=${row}") + assertTrue(row[2] != null, + "_last_updated_sequence_number should be non-null for Spark DV fixture, row=${row}") + } + def sparkChecksumRows = spark_iceberg(""" + select count(*), sum(id) + from demo.format_v3.dv_test + """) + def dorisChecksumRows = sql(""" + select count(*), sum(id) + from format_v3.dv_test + """) + assertSparkDorisResultEquals(sparkChecksumRows, dorisChecksumRows) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + try { + assertSparkGeneratedDvReadable() + spark_iceberg """create database if not exists demo.${dbName}""" + formats.each { format -> + partitionFlags.each { partitioned -> + String partitionSuffix = partitioned ? "part" : "unpart" + String sparkDvTable = "spark_dv_${format}_${partitionSuffix}" + String dorisDvTable = "doris_dv_${format}_${partitionSuffix}" + String sparkPartitionClause = partitioned ? "partitioned by (days(dt))" : "" + String dorisPartitionClause = partitioned ? "partition by list (day(dt)) ()" : "" + + try { + spark_iceberg_multi """ + drop table if exists demo.${dbName}.${sparkDvTable}; + create table demo.${dbName}.${sparkDvTable} ( + id int, + name string, + score int, + dt date + ) using iceberg + ${sparkPartitionClause} + tblproperties ( + 'format-version' = '3', + 'write.format.default' = '${format}', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read', + 'write.distribution-mode' = 'none', + 'write.delete.granularity' = 'file' + ); + set spark.sql.shuffle.partitions = 1; + set spark.sql.adaptive.enabled = false; + insert into demo.${dbName}.${sparkDvTable} + select /*+ REPARTITION(1) */ + id, + name, + score, + dt + from values + (1, 'a', 10, date '2024-09-01'), + (2, 'b', 20, date '2024-09-01'), + (3, 'c', 30, date '2024-09-02'), + (4, 'd', 40, date '2024-09-02') + as t(id, name, score, dt); + delete from demo.${dbName}.${sparkDvTable} where id in (2, 4); + reset spark.sql.shuffle.partitions; + reset spark.sql.adaptive.enabled; + """ + assertSparkDorisBusinessRows(sparkDvTable) + assertDorisLineageReadable(sparkDvTable) + assertPuffinDeleteFiles(sparkDvTable) + assertSparkDorisAggregateRows(sparkDvTable) + + sql """drop table if exists ${dorisDvTable}""" + sql """ + create table ${dorisDvTable} ( + id int, + name string, + score int, + dt date + ) engine=iceberg + ${dorisPartitionClause} + properties ( + "format-version" = "3", + "write.format.default" = "${format}", + "write.delete.mode" = "merge-on-read" + ) + """ + sql """ + insert into ${dorisDvTable} values + (1, 'a', 10, date '2024-09-01'), + (2, 'b', 20, date '2024-09-01'), + (3, 'c', 30, date '2024-09-02') + """ + sql """delete from ${dorisDvTable} where id = 2""" + assertDorisBusinessRows(dorisDvTable, [ + ["1", "a", "10", "2024-09-01"], + ["3", "c", "30", "2024-09-02"] + ]) + assertDorisLineageReadable(dorisDvTable) + assertPuffinDeleteFiles(dorisDvTable) + assertDorisAggregateRows(dorisDvTable, [["2", "4", "40"]]) + } finally { + sql """drop table if exists ${dorisDvTable}""" + sql """drop table if exists ${sparkDvTable}""" + } + } + } + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_time_travel.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_time_travel.groovy new file mode 100644 index 00000000000000..edda94be5a5316 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_time_travel.groovy @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_row_lineage_time_travel", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_row_lineage_time_travel" + String dbName = "test_row_lineage_time_travel_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + + def formats = ["parquet", "orc"] + def partitionFlags = [false, true] + + def latestSnapshot = { tableName -> + def rows = sql("""select snapshot_id, committed_at from ${tableName}\$snapshots order by committed_at""") + assertTrue(rows.size() > 0, "Snapshots should exist for ${tableName}") + return rows[rows.size() - 1] + } + + def timeTravelTimestampAfterSnapshot = { + sleep(1100) + def rows = sql("""select date_format(now(), '%Y-%m-%d %H:%i:%s')""") + return rows[0][0].toString() + } + + def businessRows = { tableName, suffix -> + def rows = sql(""" + select id, name, score + from ${tableName} ${suffix} + order by id + """) + return rows.collect { row -> [row[0].toString().toInteger(), row[1].toString(), row[2].toString().toInteger()] } + } + + def sparkBusinessRows = { tableName, suffix -> + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def rows = spark_iceberg(""" + select id, name, score + from demo.${dbName}.${tableName} ${suffix} + order by id + """) + return rows.collect { row -> [row[0].toString().toInteger(), row[1].toString(), row[2].toString().toInteger()] } + } + + def assertDorisSparkBusinessRows = { tableName, dorisSuffix, sparkSuffix, expectedRows -> + sql """refresh table ${dbName}.${tableName}""" + def dorisRows = businessRows(tableName, dorisSuffix) + def sparkRows = sparkBusinessRows(tableName, sparkSuffix) + log.info("Doris business rows for ${tableName} ${dorisSuffix}: ${dorisRows}") + log.info("Spark business rows for ${tableName} ${sparkSuffix}: ${sparkRows}") + assertEquals(dorisRows, sparkRows) + assertEquals(expectedRows, dorisRows) + return dorisRows + } + + def lineageRows = { tableName, suffix -> + def rows = sql(""" + select id, _row_id, _last_updated_sequence_number + from ${tableName} ${suffix} + order by id + """) + Map> result = [:] + rows.each { row -> + assertTrue(row[1] != null, "_row_id should be non-null for ${tableName} ${suffix}, row=${row}") + assertTrue(row[2] != null, + "_last_updated_sequence_number should be non-null for ${tableName} ${suffix}, row=${row}") + result[row[0].toString().toInteger()] = [row[1].toString().toLong(), row[2].toString().toLong()] + } + return result + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + try { + formats.each { format -> + partitionFlags.each { partitioned -> + String partitionSuffix = partitioned ? "part" : "unpart" + String tableName = "tt_${format}_${partitionSuffix}" + String partitionClause = partitioned ? "partition by list (day(dt)) ()" : "" + try { + sql """drop table if exists ${tableName}""" + sql """ + create table ${tableName} ( + id int, + name string, + score int, + dt date + ) engine=iceberg + ${partitionClause} + properties ( + "format-version" = "3", + "write.format.default" = "${format}", + "write.update.mode" = "merge-on-read", + "write.delete.mode" = "merge-on-read" + ) + """ + + sql """ + insert into ${tableName} values + (101, 'tt_a', 10, date '2024-04-01'), + (102, 'tt_b', 20, date '2024-04-02') + """ + def insertSnapshot = latestSnapshot(tableName) + String insertTimeTravel = timeTravelTimestampAfterSnapshot() + + sql """update ${tableName} set score = 21 where id = 102""" + def updateSnapshot = latestSnapshot(tableName) + String updateTimeTravel = timeTravelTimestampAfterSnapshot() + + sql """delete from ${tableName} where id = 101""" + def deleteSnapshot = latestSnapshot(tableName) + + def insertVersionRows = assertDorisSparkBusinessRows( + tableName, + "for version as of ${insertSnapshot[0]}", + "VERSION AS OF ${insertSnapshot[0]}", + [ + [101, "tt_a", 10], + [102, "tt_b", 20] + ]) + Map> insertLineage = lineageRows( + tableName, + "for version as of ${insertSnapshot[0]}") + + assertDorisSparkBusinessRows( + tableName, + "for version as of ${updateSnapshot[0]}", + "VERSION AS OF ${updateSnapshot[0]}", + [ + [101, "tt_a", 10], + [102, "tt_b", 21] + ]) + Map> updateLineage = lineageRows( + tableName, + "for version as of ${updateSnapshot[0]}") + assertEquals(insertLineage[102][0], updateLineage[102][0]) + assertTrue(updateLineage[102][1] > insertLineage[102][1], + "Time-travel UPDATE snapshot should advance sequence for id=102 in ${tableName}") + + assertDorisSparkBusinessRows( + tableName, + "for version as of ${deleteSnapshot[0]}", + "VERSION AS OF ${deleteSnapshot[0]}", + [ + [102, "tt_b", 21] + ]) + + def insertTimeRows = assertDorisSparkBusinessRows( + tableName, + "for time as of '${insertTimeTravel}'", + "TIMESTAMP AS OF '${insertTimeTravel}'", + [ + [101, "tt_a", 10], + [102, "tt_b", 20] + ]) + assertEquals(insertVersionRows, insertTimeRows) + assertDorisSparkBusinessRows( + tableName, + "for time as of '${updateTimeTravel}'", + "TIMESTAMP AS OF '${updateTimeTravel}'", + [ + [101, "tt_a", 10], + [102, "tt_b", 21] + ]) + Map> updateTimeLineage = lineageRows( + tableName, + "for time as of '${updateTimeTravel}'") + assertEquals(updateLineage[102][0], updateTimeLineage[102][0]) + assertEquals(updateLineage[102][1], updateTimeLineage[102][1]) + } finally { + sql """drop table if exists ${tableName}""" + } + } + } + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_uniqueness_stability.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_uniqueness_stability.groovy new file mode 100644 index 00000000000000..24eb447357634f --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_uniqueness_stability.groovy @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_row_lineage_uniqueness_stability", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_row_lineage_uniqueness_stability" + String dbName = "test_row_lineage_uniqueness_stability_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + + def formats = ["parquet", "orc"] + def partitionFlags = [false, true] + + def lineageMap = { tableName -> + def rows = sql(""" + select id, _row_id, _last_updated_sequence_number + from ${tableName} + order by id + """) + Map> result = [:] + rows.each { row -> + assertTrue(row[1] != null, "_row_id should be non-null for ${tableName}, row=${row}") + assertTrue(row[2] != null, + "_last_updated_sequence_number should be non-null for ${tableName}, row=${row}") + result[row[0].toString().toInteger()] = [row[1].toString().toLong(), row[2].toString().toLong()] + } + return result + } + + def assertUniqueRowIds = { tableName -> + def counts = sql(""" + select count(*), count(distinct _row_id) + from ${tableName} + where _row_id is not null + """) + log.info("Checking row_id uniqueness for ${tableName}: ${counts}") + assertEquals(counts[0][0].toString().toLong(), counts[0][1].toString().toLong()) + } + + def assertSparkDorisBusinessRows = { tableName, expectedRows -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select id, name, score, dt + from demo.${dbName}.${tableName} + order by id + """) + def dorisRows = sql(""" + select id, name, score, dt + from ${tableName} + order by id + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + + def normalizedRows = dorisRows.collect { + [it[0].toString().toInteger(), it[1].toString(), it[2].toString().toInteger(), it[3].toString()] + } + assertEquals(expectedRows, normalizedRows) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + try { + formats.each { format -> + partitionFlags.each { partitioned -> + String partitionSuffix = partitioned ? "part" : "unpart" + String tableName = "unique_stable_${format}_${partitionSuffix}" + String partitionClause = partitioned ? "partition by list (day(dt)) ()" : "" + try { + sql """drop table if exists ${tableName}""" + sql """ + create table ${tableName} ( + id int, + name string, + score int, + dt date + ) engine=iceberg + ${partitionClause} + properties ( + "format-version" = "3", + "write.format.default" = "${format}", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """insert into ${tableName} values (1, 'a', 10, date '2024-08-01'), (2, 'b', 20, date '2024-08-02')""" + sql """insert into ${tableName} values (3, 'c', 30, date '2024-08-03'), (4, 'd', 40, date '2024-08-04')""" + sql """insert into ${tableName} values (5, 'e', 50, date '2024-08-05'), (6, 'f', 60, date '2024-08-06')""" + + Map> initialLineage = lineageMap(tableName) + assertUniqueRowIds(tableName) + + sql """update ${tableName} set score = score + 10 where id = 1""" + Map> afterUpdateLineage = lineageMap(tableName) + assertUniqueRowIds(tableName) + assertEquals(initialLineage[1][0], afterUpdateLineage[1][0]) + assertTrue(afterUpdateLineage[1][1] > initialLineage[1][1], + "UPDATE should advance sequence for id=1 in ${tableName}") + + sql """ + merge into ${tableName} t + using ( + select 4 as id, 'stable_m' as name, 404 as score, date '2024-08-04' as dt + ) s + on t.id = s.id + when matched then update set name = s.name, score = s.score + """ + Map> afterMergeLineage = lineageMap(tableName) + assertUniqueRowIds(tableName) + assertEquals(initialLineage[4][0], afterMergeLineage[4][0]) + assertTrue(afterMergeLineage[4][1] > initialLineage[4][1], + "MERGE UPDATE should advance sequence for id=4 in ${tableName}") + + def rewriteResult = sql(""" + alter table ${catalogName}.${dbName}.${tableName} + execute rewrite_data_files( + "target-file-size-bytes" = "10485760", + "min-input-files" = "1" + ) + """) + log.info("rewrite_data_files result for ${tableName}: ${rewriteResult}") + assertTrue(rewriteResult.size() > 0, + "rewrite_data_files should return summary rows for ${tableName}") + + Map> afterRewriteLineage = lineageMap(tableName) + assertUniqueRowIds(tableName) + assertEquals(afterUpdateLineage[1][0], afterRewriteLineage[1][0]) + assertEquals(afterMergeLineage[4][0], afterRewriteLineage[4][0]) + assertEquals(afterMergeLineage[4][1], afterRewriteLineage[4][1]) + + assertSparkDorisBusinessRows(tableName, [ + [1, "a", 20, "2024-08-01"], + [2, "b", 20, "2024-08-02"], + [3, "c", 30, "2024-08-03"], + [4, "stable_m", 404, "2024-08-04"], + [5, "e", 50, "2024-08-05"], + [6, "f", 60, "2024-08-06"] + ]) + } finally { + sql """drop table if exists ${tableName}""" + } + } + } + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_update_delete_merge.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_update_delete_merge.groovy index 6e11b651b07769..014e94278df5d6 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_update_delete_merge.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_v3_row_lineage_update_delete_merge.groovy @@ -112,6 +112,34 @@ suite("test_iceberg_v3_row_lineage_update_delete_merge", "p0,external,iceberg,ex return result } + def normalizeRows = { rows -> + return rows.collect { row -> row.collect { col -> col == null ? null : col.toString() } } + } + + def assertSparkDorisBusinessRows = { tableName, columns, orderBy, expectedRows = null -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select ${columns} + from demo.${dbName}.${tableName} + order by ${orderBy} + """) + def dorisRows = sql(""" + select ${columns} + from ${tableName} + order by ${orderBy} + """) + log.info("Spark business rows for ${tableName}: ${sparkRows}") + log.info("Doris business rows for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + + def normalizedRows = normalizeRows(dorisRows) + if (expectedRows != null) { + assertEquals(expectedRows, normalizedRows) + } + return normalizedRows + } + sql """drop catalog if exists ${catalogName}""" sql """ create catalog if not exists ${catalogName} properties ( @@ -134,6 +162,8 @@ suite("test_iceberg_v3_row_lineage_update_delete_merge", "p0,external,iceberg,ex try { formats.each { format -> String updateDeleteTable = "test_row_lineage_v3_update_delete_${format}" + String advancedUpdateDeleteTable = "test_row_lineage_v3_update_delete_adv_${format}" + String advancedSourceTable = "test_row_lineage_v3_update_delete_src_${format}" String mergeTable = "test_row_lineage_v3_merge_${format}" log.info("Run row lineage update/delete/merge test with format ${format}") @@ -168,15 +198,10 @@ suite("test_iceberg_v3_row_lineage_update_delete_merge", "p0,external,iceberg,ex // 2. DELETE removes the target row. // 3. V3 delete files use Puffin deletion vectors instead of delete_pos parquet/orc files. // 4. Explicit row lineage reads remain non-null after DML. - def updateDeleteRows = sql """select * from ${updateDeleteTable} order by id""" - log.info("Checking table rows after UPDATE/DELETE on ${updateDeleteTable}: ${updateDeleteRows}") - assertEquals(2, updateDeleteRows.size()) - assertEquals(1, updateDeleteRows[0][0].toString().toInteger()) - assertEquals("Alice_u", updateDeleteRows[0][1]) - assertEquals(26, updateDeleteRows[0][2].toString().toInteger()) - assertEquals(3, updateDeleteRows[1][0].toString().toInteger()) - assertEquals("Charlie", updateDeleteRows[1][1]) - assertEquals(35, updateDeleteRows[1][2].toString().toInteger()) + assertSparkDorisBusinessRows(updateDeleteTable, "id, name, age", "id", [ + ["1", "Alice_u", "26"], + ["3", "Charlie", "35"] + ]) assertExplicitRowLineageNonNull(updateDeleteTable, 2) def updateDeleteLineageAfter = lineageMap(updateDeleteTable) @@ -204,6 +229,93 @@ suite("test_iceberg_v3_row_lineage_update_delete_merge", "p0,external,iceberg,ex log.info("Checking _row_id filter after UPDATE/DELETE on ${updateDeleteTable}: minRowId=${minRowIdAfterUpdate}, result=${rowIdFilterResult}") assertEquals(1, rowIdFilterResult[0][0].toString().toInteger()) + sql """drop table if exists ${advancedSourceTable}""" + sql """drop table if exists ${advancedUpdateDeleteTable}""" + sql """ + create table ${advancedUpdateDeleteTable} ( + id int, + name string, + score int, + dt date + ) engine=iceberg + partition by list (day(dt)) () + properties ( + "format-version" = "3", + "write.format.default" = "${format}", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read" + ) + """ + sql """ + create table ${advancedSourceTable} ( + id int, + action string + ) engine=iceberg + properties ( + "format-version" = "3", + "write.format.default" = "${format}" + ) + """ + sql """ + insert into ${advancedUpdateDeleteTable} values + (10, 'a', 10, date '2024-02-01'), + (11, 'b', 20, date '2024-02-01'), + (12, 'c', 30, date '2024-02-02'), + (13, 'd', 40, date '2024-02-02'), + (14, 'e', 50, date '2024-02-03') + """ + sql """insert into ${advancedSourceTable} values (12, 'U'), (13, 'D')""" + + def advancedLineageBefore = lineageMap(advancedUpdateDeleteTable) + sql """ + update ${advancedUpdateDeleteTable} + set name = concat(name, '_multi'), score = score + 100 + where score >= 10 and dt = date '2024-02-01' + """ + def advancedLineageAfterMultiUpdate = lineageMap(advancedUpdateDeleteTable) + [10, 11].each { id -> + assertEquals(advancedLineageBefore[id][0], advancedLineageAfterMultiUpdate[id][0]) + assertTrue(advancedLineageAfterMultiUpdate[id][1].toLong() > advancedLineageBefore[id][1].toLong(), + "partition multi-row UPDATE should advance sequence for id=${id}") + } + assertEquals(advancedLineageBefore[12], advancedLineageAfterMultiUpdate[12]) + + sql """ + update ${advancedUpdateDeleteTable} + set name = 'sub_u', score = score + 7 + where id in (select id from ${advancedSourceTable} where action = 'U') + """ + def advancedLineageAfterSubqueryUpdate = lineageMap(advancedUpdateDeleteTable) + assertEquals(advancedLineageAfterMultiUpdate[12][0], advancedLineageAfterSubqueryUpdate[12][0]) + assertTrue(advancedLineageAfterSubqueryUpdate[12][1].toLong() + > advancedLineageAfterMultiUpdate[12][1].toLong(), + "subquery UPDATE should advance sequence for id=12") + + sql """ + delete from ${advancedUpdateDeleteTable} + where id in (select id from ${advancedSourceTable} where action = 'D') + """ + + assertSparkDorisBusinessRows(advancedUpdateDeleteTable, "id, name, score, dt", "id", [ + ["10", "a_multi", "110", "2024-02-01"], + ["11", "b_multi", "120", "2024-02-01"], + ["12", "sub_u", "37", "2024-02-02"], + ["14", "e", "50", "2024-02-03"] + ]) + def partitionRowsAfterDelete = sql """ + select id + from ${advancedUpdateDeleteTable} + where dt = date '2024-02-02' + order by id + """ + assertEquals(1, partitionRowsAfterDelete.size()) + assertEquals(12, partitionRowsAfterDelete[0][0].toString().toInteger()) + assertExplicitRowLineageNonNull(advancedUpdateDeleteTable, 4) + def advancedLineageAfterDelete = lineageMap(advancedUpdateDeleteTable) + assertTrue(!advancedLineageAfterDelete.containsKey(13), + "subquery DELETE should remove id=13 from ${advancedUpdateDeleteTable}") + assertDeleteFilesArePuffin(advancedUpdateDeleteTable) + sql """drop table if exists ${mergeTable}""" sql """ create table ${mergeTable} ( @@ -250,18 +362,11 @@ suite("test_iceberg_v3_row_lineage_update_delete_merge", "p0,external,iceberg,ex // 1. MERGE applies DELETE, UPDATE, and INSERT actions in one statement. // 2. The partitioned MERGE still writes Puffin deletion vectors. // 3. At least one current data file written by MERGE contains physical row lineage columns. - def mergeRows = sql """select * from ${mergeTable} order by id""" - log.info("Checking table rows after MERGE on ${mergeTable}: ${mergeRows}") - assertEquals(3, mergeRows.size()) - assertEquals(1, mergeRows[0][0].toString().toInteger()) - assertEquals("Penny_u", mergeRows[0][1]) - assertEquals(31, mergeRows[0][2].toString().toInteger()) - assertEquals(3, mergeRows[1][0].toString().toInteger()) - assertEquals("Rita", mergeRows[1][1]) - assertEquals(23, mergeRows[1][2].toString().toInteger()) - assertEquals(4, mergeRows[2][0].toString().toInteger()) - assertEquals("Sara", mergeRows[2][1]) - assertEquals(24, mergeRows[2][2].toString().toInteger()) + assertSparkDorisBusinessRows(mergeTable, "id, name, age, dt", "id", [ + ["1", "Penny_u", "31", "2024-01-01"], + ["3", "Rita", "23", "2024-01-03"], + ["4", "Sara", "24", "2024-01-04"] + ]) assertExplicitRowLineageNonNull(mergeTable, 3) def mergeLineageAfter = lineageMap(mergeTable) @@ -288,6 +393,8 @@ suite("test_iceberg_v3_row_lineage_update_delete_merge", "p0,external,iceberg,ex assertTrue(insertedRowLineage[0][1] != null, "Inserted MERGE row should get generated _last_updated_sequence_number") } finally { sql """drop table if exists ${mergeTable}""" + sql """drop table if exists ${advancedSourceTable}""" + sql """drop table if exists ${advancedUpdateDeleteTable}""" sql """drop table if exists ${updateDeleteTable}""" } } diff --git a/regression-test/suites/external_table_p0/iceberg_paimon_schema_time_travel_coverage.md b/regression-test/suites/external_table_p0/iceberg_paimon_schema_time_travel_coverage.md new file mode 100644 index 00000000000000..8bc1e9520b019c --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg_paimon_schema_time_travel_coverage.md @@ -0,0 +1,119 @@ +# Iceberg / Paimon Schema Evolution × Time Travel P0 Coverage + +## Scope + +This document maps the P0 regression suites that validate Doris reads after +Iceberg or Paimon schema evolution. The matrix intentionally combines schema +changes with snapshot, tag, branch, time travel, delete/upsert and reader +variants instead of testing these dimensions in isolation. + +The tests use explicit old/new field projections and negative bindings in +addition to row-shape assertions. This prevents a rename with unchanged values +from passing accidentally. + +## Schema operations + +| ID | Operation | Iceberg | Paimon | P0 contract | +| --- | --- | --- | --- | --- | +| S01 | Add one nullable top-level field | Positive | Positive | Old refs reject the new field; new refs backfill NULL | +| S02 | Add multiple top-level fields | Positive | Positive | Order, types and NULL backfill are snapshot-local | +| S03 | Reorder / FIRST / AFTER | Positive | Positive | Field IDs remain stable across old and new refs | +| S04 | Rename top-level field | Positive | Positive | Old and new names bind only in their own schemas | +| S05 | Case-only or mixed-case rename | Positive | Positive | Case handling does not hide schema selection errors | +| S06 | Drop top-level field | Positive | Positive | Old refs retain the field; current refs reject it | +| S07 | Drop and re-add the same name | Positive | Positive | Old and new field IDs never share values | +| S08 | Compatible type promotion | Positive | Positive | Historical and current types remain correct | +| S09 | Add STRUCT child | Positive | Positive | Nested field is visible only after its schema version | +| S10 | Rename STRUCT child | Positive | Positive | Nested old/new paths have snapshot-local binding | +| S11 | Drop STRUCT child | Positive | Positive | Historical nested projections remain readable | +| S12 | Drop and re-add STRUCT child | Positive | Positive | Nested field IDs never leak values | +| S13 | Promote/reorder STRUCT child | Positive | Positive | Nested predicate and projection use the right type | +| S14 | MAP value STRUCT evolution | Positive | Positive | `element_at(...).field` uses the selected schema | +| S15 | ARRAY element STRUCT evolution | Positive | Positive | `array[i].field` uses the selected schema | +| S16 | Partition-column mutation | Positive/restricted | Negative format contract | Unsupported mutations are rejected atomically | +| S17 | Partition evolution | Positive | Format-constrained contract | Data and payload evolution preserve partition reads | +| S18 | PK-table non-key evolution | N/A | Positive | Upsert/delete/DV remain correct through evolution | +| S19 | Comment/default/nullability | Positive and negative | Positive and negative | Metadata-only changes and rejections are atomic | +| S20 | Narrowing, map-key, key/partition mutations | Negative | Negative | No schema, snapshot or cached state is polluted | + +## Historical references and actions + +| ID | Reference/action | Iceberg | Paimon | Coverage | +| --- | --- | --- | --- | --- | +| T00 | Latest/current | Yes | Yes | Current schema, explicit projection, predicate, aggregate | +| T01/T02 | Pre/post numeric snapshot | Yes | Yes | Old/new field binding and row visibility | +| T03 | Timestamp string | Yes | Yes | Resolves the expected pre-change schema | +| T04 | Epoch millis | Stable rejection | Yes | Iceberg syntax rejection; Paimon positive read | +| T05/T06/T07 | Pre/post tag forms | Yes | Yes | `FOR VERSION AS OF` and `@tag` | +| T08 | Pre-change branch | Negative product contract | Yes | Branch schema/data are compared with the format oracle | +| T09 | Independent branch evolution | Negative product contract | Negative product contract | Failure is isolated without crashing the shared BE | +| T10 | Expired snapshot retained by tag | Existing lifecycle + matrix | Yes | Retained tag never falls back to latest | +| T11 | Missing snapshot/tag | Yes | Yes | Stable error, never latest fallback | +| T12/T13 | Dual historical relations | Negative product contract | Negative product contract | Join, reverse join, UNION, nested UNION, CTE, subquery | +| T14 | Incremental read across change | N/A | Yes | JNI/CPP-supported paths use the end schema | +| T15 | Rollback to snapshot/timestamp | Yes | N/A | Data rolls back while Iceberg current schema semantics remain correct | +| T16 | Cherry-pick / fast-forward | Yes | Paimon fast-forward oracle | Current, tag and branch state are verified | + +## Delete and execution dimensions + +| Dimension | Iceberg | Paimon | +| --- | --- | --- | +| Position delete | v2, Parquet and ORC, before/after evolution | N/A | +| Equality delete | Rename, promotion, drop/re-add, old/new snapshots | N/A | +| Deletion vector | v3, Parquet and ORC, before/after evolution | PK-table DV path | +| Row operations | Delete visibility around every checkpoint | Upsert, delete and compaction | +| Readers | File scanner V1/V2 | JNI/native/CPP-supported paths | +| Cache | REST cache on/off | Filesystem metadata cache on/off | +| Catalog smoke | REST full matrix and JDBC rename/time-travel | Filesystem full matrix and JDBC rename/time-travel | +| Cluster topology | External endpoints are cluster-reachable; JDBC drivers are installed on every FE/BE | External endpoints are cluster-reachable; JDBC drivers are installed on every FE/BE | + +HMS and DLF are credential/environment variants rather than additional schema +semantics. Their existing catalog suites remain responsible for connectivity; +the deterministic P0 schema matrix uses REST/filesystem, and JDBC gets an +explicit rename plus old snapshot/tag smoke path. + +## Suite map + +| Suite | Responsibility | +| --- | --- | +| `iceberg/test_iceberg_schema_time_travel_matrix.groovy` | S01-S17 × T00-T13, nested evolution, partition evolution, cache/scanner variants | +| `iceberg/test_iceberg_schema_dual_relation_matrix.groovy` | Dual-snapshot join/UNION/CTE/subquery negative contracts | +| `iceberg/test_iceberg_schema_equality_delete_time_travel.groovy` | Equality delete × rename/promotion/drop-re-add | +| `iceberg/test_iceberg_schema_position_dv_time_travel.groovy` | Position delete and DV × top-level/nested evolution, Parquet/ORC | +| `iceberg/test_iceberg_schema_ref_actions_matrix.groovy` | Rollback, cherry-pick, fast-forward and branch action semantics | +| `iceberg/test_iceberg_schema_metadata_atomicity_matrix.groovy` | Comment/default/nullability/narrowing/map-key atomicity | +| `paimon/test_paimon_schema_time_travel_matrix.groovy` | S01-S18 × T00-T14, PK upsert/delete/DV, cache/readers | +| `paimon/test_paimon_schema_dual_relation_matrix.groovy` | Dual-snapshot join/UNION/CTE/subquery negative contracts | +| `paimon/test_paimon_schema_branch_partition_matrix.groovy` | Independent branch evolution, fast-forward and partition restrictions | +| `paimon/test_paimon_schema_metadata_atomicity_matrix.groovy` | Comment/default/nullability/narrowing atomicity | +| `iceberg/test_iceberg_jdbc_catalog.groovy` | JDBC catalog rename × numeric snapshot/tag smoke | +| `paimon/test_paimon_jdbc_catalog.groovy` | JDBC catalog rename × numeric snapshot/tag smoke | + +Each Groovy file contains `Scenario` comments identifying the matrix cell under +test. Jira keys are deliberately absent from Groovy source. Product issues link +back to the exact suite, scenario and file location from Jira instead. + +## Product contracts discovered by the matrix + +| Issue | Observed contract | Negative regression location | +| --- | --- | --- | +| DORIS-27425 | Iceberg branch can use latest schema instead of branch schema | `test_iceberg_schema_time_travel_matrix.groovy`, `test_iceberg_schema_ref_actions_matrix.groovy` | +| DORIS-27427 | Iceberg dual historical relations can share the wrong schema | `test_iceberg_schema_dual_relation_matrix.groovy` | +| DORIS-27428 | Paimon dual historical relations can share the wrong schema | `test_paimon_schema_dual_relation_matrix.groovy` | +| DORIS-27433 | Paimon branch schema init fails; a post-fast-forward scan can abort BE | `test_paimon_schema_branch_partition_matrix.groovy` | +| DORIS-27434 | Doris `DESC` omits Iceberg field comments | `test_iceberg_schema_metadata_atomicity_matrix.groovy` | + +## Validation status + +- The ten REST/filesystem matrix suites pass with no failed, fatal or skipped + suite. +- The two JDBC catalog suites pass with no failed, fatal or skipped suite. +- The validation covers current and historical schema binding, nested + evolution, deletes, branches, tags, dual historical relations, metadata + atomicity, reader/cache variants, catalog variants and distributed cluster + scheduling. + +The requested schema-change × historical-operation correctness matrix has no +unimplemented P0 cell. Unsupported format operations and currently incorrect +Doris behavior are represented by stable negative regression contracts rather +than being marked as missing. diff --git a/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy b/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy index edf41cc9d18b40..8c7cff7a8886e1 100644 --- a/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy +++ b/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy @@ -116,6 +116,9 @@ suite("test_information_schema_timezone", "p0,external") { "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy new file mode 100644 index 00000000000000..4d986201fbeea4 --- /dev/null +++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_clickhouse_jdbc_v2", "p0,external") { + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String clickhousePort = context.config.otherConfigs.get("clickhouse_22_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + // Keep large drivers in the regression bucket so isolated workers do not require public egress. + String driverUrl = "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/clickhouse-jdbc-0.9.8-all.jar" + + sql """ drop catalog if exists clickhouse_v2_schema """ + sql """ create catalog clickhouse_v2_schema properties( + "type"="jdbc", + "user"="default", + "password"="123456", + "jdbc_url" = "jdbc:clickhouse://${externalEnvIp}:${clickhousePort}/doris_test?jdbc_schema_term=schema", + "driver_url" = "${driverUrl}", + "driver_class" = "com.clickhouse.jdbc.Driver" + );""" + + order_qt_clickhouse_v2_schema """ select count(*) from clickhouse_v2_schema.doris_test.type """ + order_qt_clickhouse_v2_distributed """ select count(*) from clickhouse_v2_schema.doris_test.distributed_type """ + order_qt_clickhouse_v2_materialized_view """ select count(*) from clickhouse_v2_schema.doris_test.materialized_view_type """ + sql """ drop catalog clickhouse_v2_schema """ + } +} diff --git a/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy index dea96abc9700ea..ca4085774feabe 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy @@ -44,7 +44,11 @@ suite("test_iceberg_hadoop_catalog_kerberos", "p0,external") { "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.kerberos.keytab.login.autorenewal.enabled" = "false", "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication" ); """ diff --git a/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy index 2cd57abd65128b..0ba39257b8c2c4 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy @@ -47,12 +47,17 @@ suite("test_non_catalog_kerberos", "p0,external") { "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.kerberos.keytab.login.autorenewal.enabled" = "false", "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication" ); """ sql """ switch ${hms_catalog_name} """ - sql """ use test_krb_hive_db """ + sql """ create database if not exists test_non_catalog_krb_hive_db """ + sql """ use test_non_catalog_krb_hive_db """ sql """ drop table if exists ${test_tbl_name}""" sql """ CREATE TABLE `${test_tbl_name}` ( @@ -76,6 +81,10 @@ suite("test_non_catalog_kerberos", "p0,external") { ) with HDFS ( "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// @@ -94,6 +103,10 @@ suite("test_non_catalog_kerberos", "p0,external") { FORMAT AS CSV PROPERTIES( "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// @@ -115,6 +128,10 @@ suite("test_non_catalog_kerberos", "p0,external") { "hadoop.username" = "doris", "format" = "csv", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// @@ -130,7 +147,7 @@ suite("test_non_catalog_kerberos", "p0,external") { Awaitility.await("queery-export-task-result-test").atMost(60, SECONDS).pollInterval(5, SECONDS).until( { sql """ switch ${hms_catalog_name} """ - sql """ use test_krb_hive_db """ + sql """ use test_non_catalog_krb_hive_db """ def res = sql """ show export where label = "${export_task_label}" """ if (res[0][2] == "FINISHED") { return true diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy index e2d1fd46d86a3a..46f6667364e9dd 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -37,6 +37,10 @@ suite("test_single_hive_kerberos", "p0,external") { "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab", @@ -48,52 +52,69 @@ suite("test_single_hive_kerberos", "p0,external") { ); """ sql """ switch hms_kerberos """ + sql """ CREATE DATABASE IF NOT EXISTS test_single_krb_hive_db """ + sql """ USE test_single_krb_hive_db """ + sql """ DROP TABLE IF EXISTS test_krb_hive_tbl """ + sql """ + CREATE TABLE test_krb_hive_tbl ( + id_key INT, + string_key STRING, + rate_val DOUBLE, + comment STRING + ) ENGINE = hive + PROPERTIES ("file_format" = "parquet") + """ + sql """ + INSERT INTO test_krb_hive_tbl VALUES + (1, 'a', 3.16, 'cc0'), + (2, 'b', 41.2, 'cc1'), + (3, 'c', 6.2, 'cc2'), + (4, 'd', 1.4, 'cc3') + """ sql """ show databases """ - order_qt_q01 """ select * from hms_kerberos.test_krb_hive_db.test_krb_hive_tbl """ + order_qt_q01 """ select * from hms_kerberos.test_single_krb_hive_db.test_krb_hive_tbl """ sql """drop catalog hms_kerberos;""" - try { - sql """drop catalog if exists hms_kerberos_hadoop_err1;""" - sql """ - CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err1 - PROPERTIES ( - "type" = "hms", - "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", - "hadoop.security.authentication" = "kerberos", - "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", - "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// - RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// - RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// - DEFAULT", - "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab" - ); - """ - sql """ switch hms_kerberos_hadoop_err1 """ + sql """drop catalog if exists hms_kerberos_hadoop_err1;""" + sql """ + CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err1 + PROPERTIES ( + "type" = "hms", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// + RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// + DEFAULT", + "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab" + ); + """ + sql """ switch hms_kerberos_hadoop_err1 """ + test { sql """ show databases """ - } catch (Exception e) { - logger.info(e.toString()) - // caused by a warning msg if enable sasl on hive but "hive.metastore.sasl.enabled" is not true: - // "set_ugi() not successful, Likely cause: new client talking to old server. Continuing without it." - assertTrue(e.toString().contains("thrift.transport.TTransportException")) + exception "thrift.transport.TTransportException" } - try { - sql """drop catalog if exists hms_kerberos_hadoop_err2;""" - sql """ - CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err2 - PROPERTIES ( - "type" = "hms", - "hive.metastore.sasl.enabled " = "true", - "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" - ); - """ - sql """ switch hms_kerberos_hadoop_err2 """ + sql """drop catalog if exists hms_kerberos_hadoop_err2;""" + sql """ + CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err2 + PROPERTIES ( + "type" = "hms", + "hive.metastore.sasl.enabled " = "true", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab", + "hive.metastore.kerberos.principal" = "hive/invalid-host@LABS.TERADATA.COM" + ); + """ + sql """ switch hms_kerberos_hadoop_err2 """ + test { sql """ show databases """ - } catch (Exception e) { - // org.apache.thrift.transport.TTransportException: GSS initiate failed - assertTrue(e.toString().contains("Could not connect to meta store using any of the URIs provided. Most recent failure: shade.doris.hive.org.apache.thrift.transport.TTransportException: GSS initiate failed")) + exception "GSS initiate failed" } // try { diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index d5db528558367c..21ceb365fd8dba 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -42,6 +42,10 @@ suite("test_two_hive_kerberos", "p0,external") { "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -61,6 +65,10 @@ suite("test_two_hive_kerberos", "p0,external") { "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9683", "fs.defaultFS" = "hdfs://${externalEnvIp}:8620", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master-2@OTHERREALM.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@OTHERREALM.COM", @@ -72,6 +80,32 @@ suite("test_two_hive_kerberos", "p0,external") { ); """ + def initializeFixture = { String catalogName -> + sql """ switch ${catalogName} """ + sql """ CREATE DATABASE IF NOT EXISTS test_krb_hive_db """ + sql """ USE test_krb_hive_db """ + sql """ DROP TABLE IF EXISTS test_krb_hive_tbl """ + sql """ + CREATE TABLE test_krb_hive_tbl ( + id_key INT, + string_key STRING, + rate_val DOUBLE, + comment STRING + ) ENGINE = hive + PROPERTIES ("file_format" = "parquet") + """ + sql """ + INSERT INTO test_krb_hive_tbl VALUES + (1, 'a', 3.16, 'cc0'), + (2, 'b', 41.2, 'cc1'), + (3, 'c', 6.2, 'cc2'), + (4, 'd', 1.4, 'cc3') + """ + } + + initializeFixture(hms_catalog_name) + initializeFixture("other_${hms_catalog_name}") + // 1. catalogA sql """switch ${hms_catalog_name};""" logger.info("switched to catalog " + hms_catalog_name) diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy index b5f233aac67515..cbb3174ea5faeb 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_jdbc_catalog.groovy @@ -63,6 +63,8 @@ suite("test_paimon_jdbc_catalog", "p0,external") { String localDriverPath = "${localDriverDir}/${driverName}" String sparkDriverPath = "/tmp/${driverName}" String sparkSeedCatalogName = "${catalogName}_seed" + // Reuse the fixture-wide Docker command so local and CI permission models behave identically. + String dockerCommand = context.config.otherConfigs.get("externalDockerCommand") ?: "docker" assertTrue(jdbcDriversDir != null && !jdbcDriversDir.isEmpty(), "jdbc_drivers_dir must be configured") @@ -90,25 +92,63 @@ suite("test_paimon_jdbc_catalog", "p0,external") { } executeCommand("mkdir -p ${localDriverDir}", false, 60) - executeCommand("mkdir -p ${jdbcDriversDir}", true, 60) if (!new File(localDriverPath).exists()) { executeCommand("/usr/bin/curl --max-time 600 ${driverDownloadUrl} --output ${localDriverPath}", true, 660) } - executeCommand("cp -f ${localDriverPath} ${jdbcDriversDir}/${driverName}", true, 60) - String sparkContainerName = executeCommand("docker ps --filter name=spark-iceberg --format {{.Names}}", false, 30) + def clusterHostIps = new ArrayList() + String[][] backends = sql """show backends""" + for (def backend in backends) { + clusterHostIps.add(backend[1]) + } + String[][] frontends = sql """show frontends""" + for (def frontend in frontends) { + clusterHostIps.add(frontend[1]) + } + clusterHostIps = clusterHostIps.unique() + + Set localHostIps = ["127.0.0.1", "localhost", "::1"] as Set + java.util.Collections.list(java.net.NetworkInterface.getNetworkInterfaces()).each { networkInterface -> + java.util.Collections.list(networkInterface.getInetAddresses()).each { address -> + localHostIps.add(address.getHostAddress().split("%")[0]) + } + } + localHostIps.add(java.net.InetAddress.getLocalHost().getHostName()) + localHostIps.add(java.net.InetAddress.getLocalHost().getCanonicalHostName()) + + for (def hostIp in clusterHostIps) { + // Scenario: every FE/BE receives the JDBC driver so metadata failover and distributed + // scan scheduling do not depend on a driver installed only on the regression runner. + if (localHostIps.contains(hostIp)) { + executeCommand("mkdir -p ${jdbcDriversDir}", true, 60) + executeCommand("cp -f ${localDriverPath} ${jdbcDriversDir}/${driverName}", true, 60) + } else { + executeCommand( + "ssh -o BatchMode=yes -o StrictHostKeyChecking=no root@${hostIp} \"mkdir -p ${jdbcDriversDir}\"", + true, + 60 + ) + scpFiles("root", hostIp, localDriverPath, jdbcDriversDir, false) + } + } + + String sparkContainerName = executeCommand( + "${dockerCommand} ps --filter name=spark-iceberg --format {{.Names}}", + false, + 30 + ) ?.trim() if (sparkContainerName == null || sparkContainerName.isEmpty()) { logger.info("spark-iceberg container not found, skip this test") return } - executeCommand("docker cp ${localDriverPath} ${sparkContainerName}:${sparkDriverPath}", true, 60) + executeCommand("${dockerCommand} cp ${localDriverPath} ${sparkContainerName}:${sparkDriverPath}", true, 60) String sparkMinioEndpoint = "http://${externalEnvIp}:${minioPort}" if (sparkContainerName.contains("spark-iceberg")) { String sparkMinioContainerName = sparkContainerName.replaceFirst("spark-iceberg", "minio") String resolvedSparkMinioContainer = executeCommand( - "docker ps --filter name=${sparkMinioContainerName} --format {{.Names}}", + "${dockerCommand} ps --filter name=${sparkMinioContainerName} --format {{.Names}}", false, 30 )?.trim() @@ -121,7 +161,7 @@ suite("test_paimon_jdbc_catalog", "p0,external") { def sparkPaimonJdbc = { String sqlText -> String escapedSql = sqlText.replaceAll('"', '\\\\"') - String command = """docker exec ${sparkContainerName} spark-sql --master spark://${sparkContainerName}:7077 \ + String command = """${dockerCommand} exec ${sparkContainerName} spark-sql --master spark://${sparkContainerName}:7077 \ --jars ${sparkDriverPath} \ --driver-class-path ${sparkDriverPath} \ --conf spark.driver.extraClassPath=${sparkDriverPath} \ @@ -225,6 +265,43 @@ suite("test_paimon_jdbc_catalog", "p0,external") { assertEquals(1, rowCount.size()) assertEquals("2", rowCount[0][0].toString()) + // Scenario TC09-JDBC: Paimon JDBC catalog preserves the old snapshot schema after rename. + String jdbcOldSnapshot = sql(""" + select snapshot_id + from paimon_jdbc_tbl\$snapshots + order by snapshot_id desc + limit 1 + """)[0][0].toString() + sparkPaimonJdbc """ + CALL ${sparkSeedCatalogName}.sys.create_tag( + table => '${dbName}.paimon_jdbc_tbl', + tag => 'jdbc_before_rename', + snapshot => ${jdbcOldSnapshot} + ) + """ + sparkPaimonJdbc """ + ALTER TABLE ${sparkSeedCatalogName}.${dbName}.paimon_jdbc_tbl + RENAME COLUMN name TO jdbc_renamed_name + """ + sql """REFRESH TABLE paimon_jdbc_tbl""" + assertEquals([[1, "alice"], [2, "bob"]], sql(""" + select id, name + from paimon_jdbc_tbl for version as of ${jdbcOldSnapshot} + order by id + """)) + assertEquals([[1, "alice"], [2, "bob"]], sql(""" + select id, name + from paimon_jdbc_tbl@tag(jdbc_before_rename) + order by id + """)) + assertEquals([[1, "alice"], [2, "bob"]], sql(""" + select id, jdbc_renamed_name from paimon_jdbc_tbl order by id + """)) + test { + sql """select name from paimon_jdbc_tbl""" + exception "Unknown column 'name'" + } + assertSystemTableReadable("paimon_jdbc_tbl\$schemas", ["schema_id"], 1) assertSystemTableReadable("paimon_jdbc_tbl\$snapshots", ["snapshot_id"], 1) [ diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.groovy new file mode 100644 index 00000000000000..f0f795477396ef --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_branch_partition_matrix.groovy @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_schema_branch_partition_matrix", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_paimon_schema_branch_partition_matrix" + String dbName = "paimon_schema_branch_partition_db" + String branchTable = "branch_schema_timeline" + String partitionTable = "partition_schema_timeline" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='paimon', + 'warehouse'='s3://warehouse/wh', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.path.style.access'='true', + 'meta.cache.paimon.table.ttl-second'='0' + ) + """ + + try { + spark_paimon_multi """ + create database if not exists paimon.${dbName}; + drop table if exists paimon.${dbName}.${branchTable}; + create table paimon.${dbName}.${branchTable} ( + id int, + old_name string, + metric int + ) using paimon + tblproperties ('file.format'='parquet'); + insert into paimon.${dbName}.${branchTable} values (1, 'base-1', 10); + call paimon.sys.create_tag( + table => '${dbName}.${branchTable}', + tag => 'branch_base' + ); + call paimon.sys.create_branch( + '${dbName}.${branchTable}', + 'schema_branch', + 'branch_base' + ); + """ + + // Scenario T09-branch-evolution: branch owns an independent rename/type/add timeline. + spark_paimon_multi """ + alter table paimon.${dbName}.`${branchTable}\$branch_schema_branch` + rename column old_name to branch_name; + alter table paimon.${dbName}.`${branchTable}\$branch_schema_branch` + alter column metric type bigint; + alter table paimon.${dbName}.`${branchTable}\$branch_schema_branch` + add column branch_only string; + insert into paimon.${dbName}.`${branchTable}\$branch_schema_branch` + values (2, 'branch-2', 6000000000, 'branch-only-2'); + """ + + // Main evolves independently after the branch is created. + spark_paimon_multi """ + alter table paimon.${dbName}.${branchTable} add column main_only string; + insert into paimon.${dbName}.${branchTable} + values (3, 'main-3', 30, 'main-only-3'); + """ + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + sql """refresh table ${branchTable}""" + + assertEquals([[1, "base-1", 10, null], [3, "main-3", 30, "main-only-3"]], sql(""" + select id, old_name, metric, main_only + from ${branchTable} + order by id + """)) + // Negative contract: Doris cannot initialize a Paimon branch with an independent schema. + test { + sql """ + select id, branch_name, metric, branch_only + from ${branchTable}@branch(schema_branch) + order by id + """ + exception "failed to initSchema" + } + test { + sql """select branch_only from ${branchTable}""" + exception "Unknown column 'branch_only'" + } + + // Scenario T09-fast-forward: branch schema and data replace main after Paimon fast-forward. + spark_paimon """ + call paimon.sys.fast_forward('${dbName}.${branchTable}', 'schema_branch') + """ + spark_paimon """refresh table paimon.${dbName}.${branchTable}""" + sql """refresh table ${branchTable}""" + assertEquals([ + [1, "base-1", 10L, null], + [2, "branch-2", 6000000000L, "branch-only-2"] + ], spark_paimon(""" + select id, branch_name, metric, branch_only + from paimon.${dbName}.${branchTable} + order by id + """)) + List fastForwardColumns = sql("""desc ${branchTable}""") + .collect { row -> row[0].toString() } + assertTrue(fastForwardColumns.containsAll( + ["id", "branch_name", "metric", "branch_only"])) + test { + sql """select old_name from ${branchTable}""" + exception "Unknown column 'old_name'" + } + + spark_paimon_multi """ + drop table if exists paimon.${dbName}.${partitionTable}; + create table paimon.${dbName}.${partitionTable} ( + id int, + part string, + old_payload string + ) using paimon + partitioned by (part) + tblproperties ('file.format'='orc'); + insert into paimon.${dbName}.${partitionTable} + values (1, 'p1', 'old-1'), (2, 'p2', 'old-2'); + call paimon.sys.create_tag( + table => '${dbName}.${partitionTable}', + tag => 'partition_before_change' + ); + alter table paimon.${dbName}.${partitionTable} + rename column old_payload to new_payload; + insert into paimon.${dbName}.${partitionTable} + values (3, 'p3', 'new-3'); + """ + sql """refresh table ${partitionTable}""" + + // Scenario S16-supported: non-partition payload rename keeps old/new snapshot bindings. + assertEquals([[1, "p1", "old-1"], [2, "p2", "old-2"]], sql(""" + select id, part, old_payload + from ${partitionTable}@tag(partition_before_change) + order by id + """)) + assertEquals([ + [1, "p1", "old-1"], + [2, "p2", "old-2"], + [3, "p3", "new-3"] + ], sql(""" + select id, part, new_payload from ${partitionTable} order by id + """)) + + // Scenario S16-negative: Paimon partition keys only support reordering. + // The operation must fail atomically without changing schema, snapshots or partition data. + int snapshotsBefore = spark_paimon(""" + select count(*) + from paimon.${dbName}.`${partitionTable}\$snapshots` + """)[0][0].toString().toInteger() + boolean renameRejected = false + try { + spark_paimon """ + alter table paimon.${dbName}.${partitionTable} + rename column part to renamed_part + """ + } catch (Exception e) { + renameRejected = true + assertTrue(e.message.toLowerCase().contains("partition")) + } + assertTrue(renameRejected, "Paimon must reject partition-key rename") + + boolean typeRejected = false + try { + spark_paimon """ + alter table paimon.${dbName}.${partitionTable} + alter column part type bigint + """ + } catch (Exception e) { + typeRejected = true + assertTrue(e.message.toLowerCase().contains("partition")) + } + assertTrue(typeRejected, "Paimon must reject partition-key type changes") + + boolean dropRejected = false + try { + spark_paimon """ + alter table paimon.${dbName}.${partitionTable} drop column part + """ + } catch (Exception e) { + dropRejected = true + assertTrue(e.message.toLowerCase().contains("partition")) + } + assertTrue(dropRejected, "Paimon must reject dropping a partition key") + + int snapshotsAfter = spark_paimon(""" + select count(*) + from paimon.${dbName}.`${partitionTable}\$snapshots` + """)[0][0].toString().toInteger() + assertEquals(snapshotsBefore, snapshotsAfter) + sql """refresh table ${partitionTable}""" + assertEquals([ + [1, "p1", "old-1"], + [2, "p2", "old-2"], + [3, "p3", "new-3"] + ], sql(""" + select id, part, new_payload from ${partitionTable} order by id + """)) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.groovy new file mode 100644 index 00000000000000..b5478dfbea7348 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_dual_relation_matrix.groovy @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_schema_dual_relation_matrix", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_paimon_schema_dual_relation_matrix" + String dbName = "paimon_schema_dual_relation_db" + String tableName = "dual_schema_timeline" + + def latestSnapshotId = { + List> rows = spark_paimon """ + select snapshot_id + from paimon.${dbName}.`${tableName}\$snapshots` + order by snapshot_id desc + limit 1 + """ + assertEquals(1, rows.size()) + return rows[0][0].toString() + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='paimon', + 'warehouse'='s3://warehouse/wh', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.path.style.access'='true' + ) + """ + + try { + spark_paimon_multi """ + create database if not exists paimon.${dbName}; + drop table if exists paimon.${dbName}.${tableName}; + create table paimon.${dbName}.${tableName} ( + id int, + old_name string, + info struct + ) using paimon + tblproperties ('file.format'='parquet'); + insert into paimon.${dbName}.${tableName} + values (1, 'old-1', named_struct('added', 10, 'keep', 11)); + """ + String oldSnapshot = latestSnapshotId() + + spark_paimon_multi """ + alter table paimon.${dbName}.${tableName} rename column old_name to new_name; + alter table paimon.${dbName}.${tableName} + rename column info.added to renamed; + insert into paimon.${dbName}.${tableName} + values (2, 'new-2', named_struct('renamed', 20, 'keep', 21)); + """ + String newSnapshot = latestSnapshotId() + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + // Scenario TC07-baseline: verify a single historical relation binds its own schema. + assertEquals([[1, "old-1", 10]], sql(""" + select id, old_name, info.added + from ${tableName} for version as of ${oldSnapshot} + order by id + """)) + assertEquals([[1, "old-1", 10], [2, "new-2", 20]], sql(""" + select id, new_name, info.renamed + from ${tableName} for version as of ${newSnapshot} + order by id + """)) + + // Scenario TC07-join negative contract: + // two Paimon historical relations currently reuse the first schema. + test { + sql """ + select o.id, o.old_name, n.new_name + from ( + select id, old_name + from ${tableName} for version as of ${oldSnapshot} + ) o + join ( + select id, new_name + from ${tableName} for version as of ${newSnapshot} + ) n on o.id = n.id + order by o.id + """ + exception "Unknown column 'new_name'" + } + + // Scenario TC07-reverse-join: binding must be independent of relation order. + test { + sql """ + select n.id, n.new_name, o.old_name + from ( + select id, new_name + from ${tableName} for version as of ${newSnapshot} + ) n + join ( + select id, old_name + from ${tableName} for version as of ${oldSnapshot} + ) o on n.id = o.id + order by n.id + """ + exception "Unknown column 'old_name'" + } + + // Scenario TC07-union: top-level historical schemas stay relation-local. + test { + sql """ + select id, old_name as name_value + from ${tableName} for version as of ${oldSnapshot} + union all + select id, new_name as name_value + from ${tableName} for version as of ${newSnapshot} + order by id, name_value + """ + exception "Unknown column 'new_name'" + } + + // Scenario TC07-nested-union: nested lookup is also relation-local. + test { + sql """ + select id, info.added as nested_value + from ${tableName} for version as of ${oldSnapshot} + union all + select id, info.renamed as nested_value + from ${tableName} for version as of ${newSnapshot} + order by id, nested_value + """ + exception "No such struct field 'renamed'" + } + + // Scenario TC07-CTE: CTE boundaries must not collapse snapshot schemas. + test { + sql """ + with old_ref as ( + select id, old_name + from ${tableName} for version as of ${oldSnapshot} + ), new_ref as ( + select id, new_name + from ${tableName} for version as of ${newSnapshot} + ) + select old_ref.id, old_ref.old_name, new_ref.new_name + from old_ref join new_ref on old_ref.id = new_ref.id + order by old_ref.id + """ + exception "Unknown column 'new_name'" + } + + // Scenario TC07-correlated-subquery: subqueries require an independent schema. + test { + sql """ + select o.id, o.old_name + from ${tableName} for version as of ${oldSnapshot} o + where exists ( + select 1 + from ${tableName} for version as of ${newSnapshot} n + where n.id = o.id and n.new_name is not null + ) + order by o.id + """ + exception "Unknown column 'new_name'" + } + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_metadata_atomicity_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_metadata_atomicity_matrix.groovy new file mode 100644 index 00000000000000..5672607cf1750a --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_metadata_atomicity_matrix.groovy @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_schema_metadata_atomicity_matrix", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_paimon_schema_metadata_atomicity_matrix" + String dbName = "paimon_schema_metadata_atomicity_db" + String tableName = "metadata_timeline" + + def createTag = { String tagName -> + spark_paimon """ + call paimon.sys.create_tag( + table => '${dbName}.${tableName}', + tag => '${tagName}' + ) + """ + } + def snapshotCount = { + return spark_paimon(""" + select count(*) from paimon.${dbName}.`${tableName}\$snapshots` + """)[0][0].toString().toInteger() + } + def schemaCount = { + return spark_paimon(""" + select count(*) from paimon.${dbName}.`${tableName}\$schemas` + """)[0][0].toString().toInteger() + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='paimon', + 'warehouse'='s3://warehouse/wh', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.path.style.access'='true', + 'meta.cache.paimon.table.ttl-second'='0' + ) + """ + + try { + spark_paimon_multi """ + create database if not exists paimon.${dbName}; + drop table if exists paimon.${dbName}.${tableName}; + create table paimon.${dbName}.${tableName} ( + id int not null, + required_name string not null, + optional_value int, + info struct + ) using paimon + tblproperties ('file.format'='parquet'); + insert into paimon.${dbName}.${tableName} + values (1, 'name-1', 10, named_struct('value', 100, 'keep', 101)); + """ + createTag("metadata_before_change") + int initialSnapshots = snapshotCount() + + // Scenario S19-comment: top-level and nested comments create schema versions, not data snapshots. + spark_paimon """ + alter table paimon.${dbName}.${tableName} + alter column required_name comment 'top-level-comment' + """ + spark_paimon """ + alter table paimon.${dbName}.${tableName} + alter column info.value comment 'nested-comment' + """ + assertEquals(initialSnapshots, snapshotCount()) + + // Scenario S19-nullability: required-to-optional keeps both current and tagged reads stable. + spark_paimon """ + alter table paimon.${dbName}.${tableName} + alter column required_name drop not null + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + sql """refresh table ${tableName}""" + assertEquals([[1, "name-1", 10, 100]], sql(""" + select id, required_name, optional_value, info.value + from ${tableName} + order by id + """)) + assertEquals([[1, "name-1", 10, 100]], sql(""" + select id, required_name, optional_value, info.value + from ${tableName}@tag(metadata_before_change) + order by id + """)) + + // Scenario S20-default: defaults are schema changes and apply when a later insert omits the field. + int schemasBeforeDefault = schemaCount() + spark_paimon """ + alter table paimon.${dbName}.${tableName} + alter column optional_value set default 7 + """ + assertTrue(schemaCount() > schemasBeforeDefault) + spark_paimon """ + insert into paimon.${dbName}.${tableName} (id, required_name, info) + values (2, 'name-2', named_struct('value', 200, 'keep', 201)) + """ + int snapshotsAfterDefault = snapshotCount() + sql """refresh table ${tableName}""" + assertEquals([[1, 10], [2, 7]], sql(""" + select id, optional_value from ${tableName} order by id + """)) + + // Scenario S20-nullability-strengthen: Paimon rejects optional-to-required atomically. + int schemasBeforeNotNull = schemaCount() + boolean notNullRejected = false + try { + spark_paimon """ + alter table paimon.${dbName}.${tableName} + alter column optional_value set not null + """ + } catch (Exception e) { + notNullRejected = true + assertTrue(e.message.contains("Cannot change nullable column to non-nullable")) + } + assertTrue(notNullRejected, "Paimon must reject optional-to-required evolution") + assertEquals(schemasBeforeNotNull, schemaCount()) + + // Scenario S20-narrowing: incompatible narrowing is atomic. + int schemasBeforeNarrowing = schemaCount() + boolean narrowingRejected = false + try { + spark_paimon """ + alter table paimon.${dbName}.${tableName} + alter column optional_value type smallint + """ + } catch (Exception e) { + narrowingRejected = true + assertTrue(e.message.toLowerCase().contains("type")) + } + assertTrue(narrowingRejected, "Paimon must reject incompatible narrowing") + assertEquals(schemasBeforeNarrowing, schemaCount()) + + sql """refresh table ${tableName}""" + assertEquals([ + [1, "name-1", 10, 100], + [2, "name-2", 7, 200] + ], sql(""" + select id, required_name, optional_value, info.value + from ${tableName} + order by id + """)) + assertEquals(snapshotsAfterDefault, snapshotCount()) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy new file mode 100644 index 00000000000000..7ec5a571a211df --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy @@ -0,0 +1,600 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_paimon_schema_time_travel_matrix" + String noCacheCatalogName = "test_paimon_schema_time_travel_matrix_no_cache" + String dbName = "paimon_schema_time_travel_matrix_db" + String topTable = "top_timeline" + String nestedTable = "nested_timeline" + String pkTable = "pk_dv_timeline" + String partitionTable = "partition_timeline" + + def latestSnapshotId = { String tableName -> + List> rows = spark_paimon """ + SELECT snapshot_id + FROM paimon.${dbName}.`${tableName}\$snapshots` + ORDER BY snapshot_id DESC + LIMIT 1 + """ + assertEquals(1, rows.size()) + return rows[0][0].toString() + } + + def createTag = { String tableName, String tagName, String snapshotId -> + spark_paimon """ + CALL paimon.sys.create_tag( + table => '${dbName}.${tableName}', + tag => '${tagName}', + snapshot => ${snapshotId} + ) + """ + } + + def assertUnknownColumn = { String query, String columnName -> + test { + sql query + exception "'${columnName}'" + } + } + + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${noCacheCatalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type'='paimon', + 'warehouse'='s3://warehouse/wh', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.path.style.access'='true' + ) + """ + sql """ + CREATE CATALOG ${noCacheCatalogName} PROPERTIES ( + 'type'='paimon', + 'warehouse'='s3://warehouse/wh', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.path.style.access'='true', + 'meta.cache.paimon.table.ttl-second'='0' + ) + """ + + try { + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.${topTable}; + CREATE TABLE paimon.${dbName}.${topTable} ( + id INT, + old_name STRING, + victim STRING, + metric INT + ) USING paimon + TBLPROPERTIES ('file.format'='parquet'); + INSERT INTO paimon.${dbName}.${topTable} + VALUES (1, 'alpha', 'old-v1', 10); + """ + String topCp0 = latestSnapshotId(topTable) + createTag(topTable, "top_cp0", topCp0) + spark_paimon """ + CALL paimon.sys.create_branch( + '${dbName}.${topTable}', + 'top_cp0_branch', + 'top_cp0' + ) + """ + Thread.sleep(1100) + + // Scenario S01/S02/S03 x T00/T01/T02/T05/T08: + // add multiple columns and position one AFTER old_name. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${topTable} + ADD COLUMNS (added STRING AFTER old_name, added_second BIGINT); + INSERT INTO paimon.${dbName}.${topTable} + (id, old_name, victim, metric, added, added_second) + VALUES (2, 'beta', 'old-v2', 20, 'added-v2', 200); + """ + String topCpAdd = latestSnapshotId(topTable) + createTag(topTable, "top_cp_add", topCpAdd) + Thread.sleep(1100) + + // Scenario S04/S05 x T00-T08: + // explicit old/new column binding catches rename regressions hidden by SELECT *. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${topTable} + RENAME COLUMN old_name TO MixedName; + INSERT INTO paimon.${dbName}.${topTable} + (id, MixedName, victim, metric, added, added_second) + VALUES (3, 'gamma', 'old-v3', 30, 'added-v3', 300); + """ + String topCpRename = latestSnapshotId(topTable) + createTag(topTable, "top_cp_rename", topCpRename) + Thread.sleep(1100) + + // Scenario S06 x T00/T01/T02/T05: the dropped field remains available only to old refs. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${topTable} DROP COLUMN victim; + INSERT INTO paimon.${dbName}.${topTable} + (id, MixedName, metric, added, added_second) + VALUES (4, 'delta', 40, 'added-v4', 400); + """ + String topCpDrop = latestSnapshotId(topTable) + createTag(topTable, "top_cp_drop", topCpDrop) + Thread.sleep(1100) + + // Scenario S07 x T00/T01/T02/T05/T12/T13: + // drop/re-add with a different type creates a new field ID and must not expose old values. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${topTable} ADD COLUMN victim BIGINT; + INSERT INTO paimon.${dbName}.${topTable} + (id, MixedName, metric, added, added_second, victim) + VALUES (5, 'epsilon', 50, 'added-v5', 500, 5000); + """ + String topCpReadd = latestSnapshotId(topTable) + createTag(topTable, "top_cp_readd", topCpReadd) + Thread.sleep(1100) + + // Scenario S08 x T00/T01/T02/T05: compatible INT -> BIGINT promotion. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${topTable} ALTER COLUMN metric TYPE BIGINT; + INSERT INTO paimon.${dbName}.${topTable} + (id, MixedName, metric, added, added_second, victim) + VALUES (6, 'zeta', 6000000000, 'added-v6', 600, 6000); + """ + String topCpPromote = latestSnapshotId(topTable) + createTag(topTable, "top_cp_promote", topCpPromote) + + spark_paimon_multi """ + DROP TABLE IF EXISTS paimon.${dbName}.${nestedTable}; + CREATE TABLE paimon.${dbName}.${nestedTable} ( + id INT, + payload STRUCT, + attributes MAP>, + events ARRAY> + ) USING paimon + TBLPROPERTIES ('file.format'='orc'); + INSERT INTO paimon.${dbName}.${nestedTable} VALUES ( + 1, + named_struct('old_child', 10, 'keep', 11), + map('a', named_struct('old_child', 20, 'keep', 21)), + array(named_struct('old_child', 30, 'keep', 31)) + ); + """ + String nestedCp0 = latestSnapshotId(nestedTable) + createTag(nestedTable, "nested_cp0", nestedCp0) + + // Scenario S09/S14/S15 x T00/T01/T02/T05: + // add children to STRUCT, MAP value struct and ARRAY element struct. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${nestedTable} ADD COLUMN payload.added_child INT; + ALTER TABLE paimon.${dbName}.${nestedTable} + ADD COLUMN attributes.value.added_child INT; + ALTER TABLE paimon.${dbName}.${nestedTable} + ADD COLUMN events.element.added_child INT; + INSERT INTO paimon.${dbName}.${nestedTable} VALUES ( + 2, + named_struct('old_child', 110, 'keep', 111, 'added_child', 112), + map('a', named_struct('old_child', 120, 'keep', 121, 'added_child', 122)), + array(named_struct('old_child', 130, 'keep', 131, 'added_child', 132)) + ); + """ + String nestedCpAdd = latestSnapshotId(nestedTable) + + // Scenario S10/S14/S15 x T00/T01/T02/T05: rename every supported nested path. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${nestedTable} + RENAME COLUMN payload.old_child TO renamed_child; + ALTER TABLE paimon.${dbName}.${nestedTable} + RENAME COLUMN attributes.value.old_child TO renamed_child; + ALTER TABLE paimon.${dbName}.${nestedTable} + RENAME COLUMN events.element.old_child TO renamed_child; + INSERT INTO paimon.${dbName}.${nestedTable} VALUES ( + 3, + named_struct('renamed_child', 210, 'keep', 211, 'added_child', 212), + map('a', named_struct('renamed_child', 220, 'keep', 221, 'added_child', 222)), + array(named_struct('renamed_child', 230, 'keep', 231, 'added_child', 232)) + ); + """ + String nestedCpRename = latestSnapshotId(nestedTable) + createTag(nestedTable, "nested_cp_rename", nestedCpRename) + + // Scenario S11/S12/S13 x T00/T01/T02/T05: + // nested drop/re-add and type promotion preserve field-ID isolation. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${nestedTable} DROP COLUMN payload.renamed_child; + ALTER TABLE paimon.${dbName}.${nestedTable} + DROP COLUMN attributes.value.renamed_child; + ALTER TABLE paimon.${dbName}.${nestedTable} + DROP COLUMN events.element.renamed_child; + ALTER TABLE paimon.${dbName}.${nestedTable} ADD COLUMN payload.renamed_child BIGINT; + ALTER TABLE paimon.${dbName}.${nestedTable} + ADD COLUMN attributes.value.renamed_child BIGINT; + ALTER TABLE paimon.${dbName}.${nestedTable} + ADD COLUMN events.element.renamed_child BIGINT; + ALTER TABLE paimon.${dbName}.${nestedTable} + ALTER COLUMN payload.keep TYPE BIGINT; + INSERT INTO paimon.${dbName}.${nestedTable} VALUES ( + 4, + named_struct('keep', 311, 'added_child', 312, 'renamed_child', 3100), + map('a', named_struct('keep', 321, 'added_child', 322, 'renamed_child', 3200)), + array(named_struct('keep', 331, 'added_child', 332, 'renamed_child', 3300)) + ); + """ + String nestedCpReadd = latestSnapshotId(nestedTable) + + spark_paimon_multi """ + DROP TABLE IF EXISTS paimon.${dbName}.${pkTable}; + CREATE TABLE paimon.${dbName}.${pkTable} ( + id INT NOT NULL, + old_name STRING, + note STRING, + score INT + ) USING paimon + TBLPROPERTIES ( + 'bucket'='1', + 'primary-key'='id', + 'file.format'='parquet', + 'deletion-vectors.enabled'='true' + ); + INSERT INTO paimon.${dbName}.${pkTable} VALUES + (1, 'alpha', 'old-note-1', 10), + (2, 'beta', 'old-note-2', 20); + """ + String pkCp0 = latestSnapshotId(pkTable) + createTag(pkTable, "pk_cp0", pkCp0) + + // Scenario S01/S04/S18 x TC05: + // PK remains stable while a non-key field is added/renamed and rows are upserted/deleted. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${pkTable} ADD COLUMN extra STRING; + ALTER TABLE paimon.${dbName}.${pkTable} RENAME COLUMN old_name TO full_name; + INSERT INTO paimon.${dbName}.${pkTable} + (id, full_name, note, score, extra) + VALUES (1, 'alpha-updated', 'new-note-1', 11, 'extra-1'); + DELETE FROM paimon.${dbName}.${pkTable} WHERE id = 2; + """ + String pkCpRenameDelete = latestSnapshotId(pkTable) + createTag(pkTable, "pk_cp_rename_delete", pkCpRenameDelete) + + // Scenario S06/S07/S08/S18 x TC05: + // drop/re-add and promotion are combined with a later upsert and DV compaction. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${pkTable} DROP COLUMN note; + ALTER TABLE paimon.${dbName}.${pkTable} ADD COLUMN note BIGINT; + ALTER TABLE paimon.${dbName}.${pkTable} ALTER COLUMN score TYPE BIGINT; + INSERT INTO paimon.${dbName}.${pkTable} + (id, full_name, score, extra, note) + VALUES (3, 'gamma', 3000000000, 'extra-3', 3000); + """ + String pkCpReadd = latestSnapshotId(pkTable) + createTag(pkTable, "pk_cp_readd", pkCpReadd) + spark_paimon """ + CALL paimon.sys.compact(table => '${dbName}.${pkTable}', compact_strategy => 'full') + """ + + spark_paimon_multi """ + DROP TABLE IF EXISTS paimon.${dbName}.${partitionTable}; + CREATE TABLE paimon.${dbName}.${partitionTable} ( + id INT, + old_partition STRING, + payload STRING + ) USING paimon + PARTITIONED BY (old_partition) + TBLPROPERTIES ('file.format'='parquet'); + INSERT INTO paimon.${dbName}.${partitionTable} + VALUES (1, 'p1', 'old'), (2, 'p2', 'old'); + """ + String partitionCp0 = latestSnapshotId(partitionTable) + createTag(partitionTable, "partition_cp0", partitionCp0) + + // Scenario S16-negative: Paimon must reject partition-column rename atomically. + String partitionRenameError = null + try { + spark_paimon """ + ALTER TABLE paimon.${dbName}.${partitionTable} + RENAME COLUMN old_partition TO new_partition + """ + } catch (Exception e) { + partitionRenameError = e.getMessage() + } + assertNotNull(partitionRenameError) + assertTrue(partitionRenameError.contains("Cannot rename partition column")) + + // Scenario S17 x TC03: rename a payload field while retaining partition pruning. + spark_paimon_multi """ + ALTER TABLE paimon.${dbName}.${partitionTable} + RENAME COLUMN payload TO new_payload; + INSERT INTO paimon.${dbName}.${partitionTable} + VALUES (3, 'p3', 'new'); + """ + String partitionCpRename = latestSnapshotId(partitionTable) + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + sql """refresh catalog ${catalogName}""" + + // Scenario TC01: validate latest schema/data, explicit new binding, predicate and aggregate. + assertEquals([[1, null], [2, null], [3, null], [4, null], [5, 5000L], [6, 6000L]], + sql("""select id, victim from ${topTable} order by id""")) + assertEquals([[6, 6000000000L]], + sql("""select id, metric from ${topTable} where metric > 5000000000""")) + assertEquals([[6L, 6000000150L]], + sql("""select count(*), sum(metric) from ${topTable}""")) + assertUnknownColumn("""select old_name from ${topTable}""", "old_name") + + // Scenario T01/T05/T06/T08: old snapshot/tag/branch uses the old schema. + List> topCp0Rows = [[1, "alpha", "old-v1", 10]] + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of ${topCp0} + order by id + """)) + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of 'top_cp0' + order by id + """)) + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable}@tag(top_cp0) + order by id + """)) + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable}@branch(top_cp0_branch) + order by id + """)) + assertUnknownColumn(""" + select MixedName from ${topTable} for version as of ${topCp0} + """, "MixedName") + + // Scenario T03/T04: time string and epoch millis resolve to the pre-change schema. + List> cp0TimeRows = sql(""" + select date_format(date_add(commit_time, interval 1 second), '%Y-%m-%d %H:%i:%s'), + cast(unix_timestamp(commit_time) * 1000 + 999 as bigint) + from ${topTable}\$snapshots + where snapshot_id = ${topCp0} + """) + assertEquals(1, cp0TimeRows.size()) + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable} for time as of "${cp0TimeRows[0][0]}" + order by id + """)) + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable} for time as of ${cp0TimeRows[0][1]} + order by id + """)) + + // Scenario S01-S08: checkpoint projections prove name/type/field-ID isolation. + assertEquals([[1, "alpha", null], [2, "beta", "added-v2"]], + sql(""" + select id, old_name, added + from ${topTable} for version as of ${topCpAdd} + order by id + """)) + assertEquals([[1, "alpha"], [2, "beta"], [3, "gamma"]], + sql(""" + select id, MixedName + from ${topTable} for version as of ${topCpRename} + where metric >= 10 + order by id + """)) + assertUnknownColumn(""" + select old_name from ${topTable} for version as of ${topCpRename} + """, "old_name") + assertUnknownColumn(""" + select victim from ${topTable} for version as of ${topCpDrop} + """, "victim") + assertEquals([[1, null], [2, null], [3, null], [4, null], [5, 5000L]], + sql(""" + select id, victim + from ${topTable} for version as of ${topCpReadd} + order by id + """)) + assertEquals([[6, 6000000000L]], + sql(""" + select id, metric + from ${topTable} for version as of ${topCpPromote} + where metric > 5000000000 + """)) + + // Scenario TC02: nested refs validate STRUCT/MAP/ARRAY projection and predicate. + assertEquals([[1, 10, 20, 30]], + sql(""" + select id, payload.old_child, + element_at(attributes, 'a').old_child, + events[1].old_child + from ${nestedTable} for version as of ${nestedCp0} + where payload.old_child = 10 + """)) + assertEquals([[1, null, null, null], [2, 112, 122, 132]], + sql(""" + select id, payload.added_child, + element_at(attributes, 'a').added_child, + events[1].added_child + from ${nestedTable} for version as of ${nestedCpAdd} + order by id + """)) + assertEquals([[1, 10, 20, 30], [2, 110, 120, 130], [3, 210, 220, 230]], + sql(""" + select id, payload.renamed_child, + element_at(attributes, 'a').renamed_child, + events[1].renamed_child + from ${nestedTable} for version as of 'nested_cp_rename' + order by id + """)) + assertUnknownColumn(""" + select payload.old_child + from ${nestedTable} for version as of ${nestedCpRename} + """, "old_child") + assertEquals([[1, null], [2, null], [3, null], [4, 3100L]], + sql(""" + select id, payload.renamed_child + from ${nestedTable} for version as of ${nestedCpReadd} + order by id + """)) + + // Scenario TC05/S18: PK upsert/delete/DV results remain correct at old and new refs. + assertEquals([[1, "alpha", "old-note-1", 10], [2, "beta", "old-note-2", 20]], + sql(""" + select id, old_name, note, score + from ${pkTable} for version as of 'pk_cp0' + order by id + """)) + assertEquals([[1, "alpha-updated", "new-note-1", 11, "extra-1"]], + sql(""" + select id, full_name, note, score, extra + from ${pkTable} for version as of ${pkCpRenameDelete} + order by id + """)) + assertEquals([[1, "alpha-updated", null], [3, "gamma", 3000L]], + sql(""" + select id, full_name, note + from ${pkTable} for version as of ${pkCpReadd} + order by id + """)) + assertUnknownColumn(""" + select old_name from ${pkTable} for version as of ${pkCpRenameDelete} + """, "old_name") + + // Scenario T14: incremental reads crossing a rename checkpoint bind the end schema. + List> incrementalJni + List> incrementalCpp + sql """set force_jni_scanner=false""" + sql """set enable_paimon_cpp_reader=false""" + incrementalJni = sql(""" + select id, full_name, score + from ${pkTable}@incr( + 'startSnapshotId'='${pkCp0}', + 'endSnapshotId'='${pkCpRenameDelete}' + ) + order by id + """) + sql """set enable_paimon_cpp_reader=true""" + incrementalCpp = sql(""" + select id, full_name, score + from ${pkTable}@incr( + 'startSnapshotId'='${pkCp0}', + 'endSnapshotId'='${pkCpRenameDelete}' + ) + order by id + """) + assertEquals(incrementalJni, incrementalCpp) + + // Scenario TC03/S16: partition pruning and renamed payloads bind to their own snapshots. + assertEquals([[1, "p1", "old"], [2, "p2", "old"]], + sql(""" + select id, old_partition, payload + from ${partitionTable} for version as of ${partitionCp0} + where old_partition = 'p1' or old_partition = 'p2' + order by id + """)) + assertEquals([[1, "p1", "old"], [2, "p2", "old"], [3, "p3", "new"]], + sql(""" + select id, old_partition, new_payload + from ${partitionTable} for version as of ${partitionCpRename} + order by id + """)) + assertUnknownColumn(""" + select new_payload + from ${partitionTable} for version as of ${partitionCp0} + """, "new_payload") + + // Scenario TC08/S20: illegal PK/partition changes fail atomically. + test { + sql """alter table ${pkTable} drop column id""" + exception "Drop column operation is not supported" + } + test { + sql """alter table ${partitionTable} drop column old_partition""" + exception "Drop column operation is not supported" + } + assertEquals([[1, "alpha-updated"], [3, "gamma"]], + sql("""select id, full_name from ${pkTable} order by id""")) + assertEquals([[1, "p1", "old"], [2, "p2", "old"], [3, "p3", "new"]], + sql("""select id, old_partition, new_payload from ${partitionTable} order by id""")) + + // Scenario TC09/R13/R17: cache, JNI and CPP paths return the same historical schema/data. + sql """switch ${noCacheCatalogName}""" + sql """use ${dbName}""" + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of ${topCp0} + order by id + """)) + sql """switch ${catalogName}""" + sql """use ${dbName}""" + sql """set enable_paimon_cpp_reader=false""" + sql """set force_jni_scanner=true""" + List> forcedJniRows = sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of ${topCp0} + order by id + """) + sql """set force_jni_scanner=false""" + sql """set enable_paimon_cpp_reader=true""" + List> cppRows = sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of ${topCp0} + order by id + """) + assertEquals(forcedJniRows, cppRows) + + // Scenario T10/T11: retained tags survive expiration; missing refs never fall back to latest. + spark_paimon """ + ALTER TABLE paimon.${dbName}.${topTable} + SET TBLPROPERTIES ('snapshot.num-retained.min'='1') + """ + spark_paimon """ + CALL paimon.sys.expire_snapshots( + table => '${dbName}.${topTable}', + retain_max => 1 + ) + """ + sql """refresh table ${topTable}""" + assertEquals(topCp0Rows, sql(""" + select id, old_name, victim, metric + from ${topTable} for version as of 'top_cp0' + order by id + """)) + test { + sql """select * from ${topTable} for version as of 9223372036854775807""" + exception "snapshot" + } + test { + sql """select * from ${topTable} for version as of 'missing_schema_tag'""" + exception "tag" + } + } finally { + sql """set enable_paimon_cpp_reader=false""" + sql """set force_jni_scanner=false""" + sql """drop catalog if exists ${catalogName}""" + sql """drop catalog if exists ${noCacheCatalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy index 385eabf7675cae..52639c8a822cb4 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy @@ -75,6 +75,9 @@ suite("refactor_params_hdfs_all_test", "p0,external") { } def hdfsNonXmlParams = "\"fs.defaultFS\" = \"hdfs://${externalEnvIp}:8520\",\n" + + "\"dfs.namenode.kerberos.principal\" = \"hdfs/hadoop-master@LABS.TERADATA.COM\",\n" + + "\"dfs.client.use.datanode.hostname\" = \"true\",\n" + + "\"hadoop.security.token.service.use_ip\" = \"false\",\n" + "\"hadoop.kerberos.min.seconds.before.relogin\" = \"5\",\n" + "\"hadoop.security.authentication\" = \"kerberos\",\n" + "\"hadoop.kerberos.principal\"=\"hive/presto-master.docker.cluster@LABS.TERADATA.COM\",\n" + @@ -316,4 +319,4 @@ suite("refactor_params_hdfs_all_test", "p0,external") { export_hdfs("hdfs://${externalEnvIp}:8520", hdfsNonXmlParams) -} \ No newline at end of file +} diff --git a/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy b/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy new file mode 100644 index 00000000000000..6fa29c321c8e72 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_file_scanner_v2_review_fixes", "p0,external") { + def backends = sql "show backends" + assertTrue(backends.size() > 0) + def backendId = backends[0][0] + def dataPath = context.config.dataPath + "/external_table_p0/tvf" + // A developer may point this at fixtures already visible to every BE (for example, a shared + // checkout mounted at /tmp). CI leaves it unset and exercises the normal per-BE distribution. + def predeployedFixturePath = context.config.otherConfigs.get("fileScannerV2FixturePath") + def remotePath = predeployedFixturePath ?: "/tmp/file_scanner_v2_review_fixes" + def fixtureNames = [ + "file_scanner_v2_empty.csv", + "file_scanner_v2_struct_0_bigint.parquet", + "file_scanner_v2_struct_1_int.parquet", + "file_scanner_v2_unsupported_time.parquet" + ] + + if (predeployedFixturePath == null) { + mkdirRemotePathOnAllBE("root", remotePath) + for (def backend : backends) { + for (def fixtureName : fixtureNames) { + scpFiles("root", backend[1], "${dataPath}/${fixtureName}", remotePath, false) + } + } + } + + sql "set enable_file_scanner_v2 = true" + + // A zero-byte CSV is a valid zero-row split when the schema is supplied explicitly. The EOF + // raised during reader preparation must skip this split instead of failing the query. + order_qt_empty_csv """ + SELECT id, name + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_empty.csv", + "backend_id" = "${backendId}", + "format" = "csv", + "csv_schema" = "id:int;name:string") + ORDER BY id + """ + + // The first file defines STRUCT, while the second stores a as INT. Localization is + // split-specific: the BIGINT file compares BIGINT values, while the old INT file safely rewrites + // the exactly representable BIGINT literal 10 to INT and compares in the physical file type. + // If a literal cannot round-trip through INT, the mapper instead casts the INT data to BIGINT. + // Repeating the multi-file read also verifies that neither predicate rewrites nor page-cache + // range state leak from one file schema into the next file or the warm scan. + def evolvedStructQuery = """ + SELECT id, col.a, col.b + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_struct_*.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + WHERE col.a = CAST(10 AS BIGINT) + ORDER BY id + """ + order_qt_struct_leaf_promotion_cold evolvedStructQuery + order_qt_struct_leaf_promotion_warm evolvedStructQuery + + // COUNT(col) sees a non-trivial mapping for the file whose STRUCT leaf is INT while the merged + // table type is BIGINT. It must fall back to the normal scan so schema-evolution casts and + // nullability checks run instead of counting footer values directly. + order_qt_count_evolved_struct """ + SELECT COUNT(col.a) + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_struct_*.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + """ + + // TIME_MILLIS is intentionally unsupported by the record reader. It must not prevent schema + // construction when only the supported id column is projected. + order_qt_unprojected_unsupported_time """ + SELECT id + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_unsupported_time.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + ORDER BY id + """ + + // A requested unsupported leaf must fail before metadata pruning. The fixture stores positive + // TIME_MILLIS values, so `unsupported_time < 0` can be eliminated from INT32 row-group + // statistics. Without the early validation, this query incorrectly returns an empty result + // instead of preserving the unsupported logical-type contract. + test { + sql """ + SELECT unsupported_time + FROM local( + "file_path" = "${remotePath}/file_scanner_v2_unsupported_time.parquet", + "backend_id" = "${backendId}", + "format" = "parquet") + WHERE unsupported_time < 0 + """ + exception "Parquet TIME with isAdjustedToUTC=true is not supported" + } + +} diff --git a/regression-test/suites/external_table_p0/tvf/test_frontends_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_frontends_tvf.groovy index 24bc167f6c24a3..3f51be591b2e08 100644 --- a/regression-test/suites/external_table_p0/tvf/test_frontends_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_frontends_tvf.groovy @@ -20,7 +20,7 @@ suite("test_frontends_tvf", "p0,external") { List> table = sql """ select * from `frontends`(); """ logger.info("${table}") assertTrue(table.size() > 0) - assertTrue(table[0].size() == 20) + assertTrue(table[0].size() == 21) List> titleNames = sql """ describe function frontends(); """ assertTrue(titleNames[0][0] == "Name") @@ -43,6 +43,7 @@ suite("test_frontends_tvf", "p0,external") { assertTrue(titleNames[17][0] == "Version") assertTrue(titleNames[18][0] == "CurrentConnected") assertTrue(titleNames[19][0] == "LiveSince") + assertTrue(titleNames[20][0] == "LocalResourceGroup") // filter columns table = sql """ select Name from `frontends`();""" @@ -65,10 +66,10 @@ suite("test_frontends_tvf", "p0,external") { def res = sql """ select count(*) from frontends() where alive = 'true'; """ assertTrue(res[0][0] > 0) - sql """ select Name, Host, EditLogPort - HttpPort, QueryPort, RpcPort, ArrowFlightSqlPort, `Role`, IsMaster, ClusterId - `Join`, Alive, ReplayedJournalId, LastHeartbeat - IsHelper, ErrMsg, Version, CurrentConnected, LiveSince from frontends(); + sql """ select Name, Host, EditLogPort, HttpPort, QueryPort, RpcPort, ArrowFlightSqlPort, + `Role`, IsMaster, ClusterId, `Join`, Alive, ReplayedJournalId, LastStartTime, + LastHeartbeat, IsHelper, ErrMsg, Version, CurrentConnected, LiveSince, + LocalResourceGroup from frontends(); """ // test exception diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_avro.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_avro.groovy deleted file mode 100644 index ec05c3a70679e0..00000000000000 --- a/regression-test/suites/external_table_p0/tvf/test_tvf_avro.groovy +++ /dev/null @@ -1,211 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_tvf_avro", "p0,external") { - - // def all_type_file = "all_type.avro"; - // def format = "avro" - - // // s3 config - // String ak = getS3AK() - // String sk = getS3SK() - // String s3_endpoint = getS3Endpoint() - // String region = getS3Region() - // String bucket = context.config.otherConfigs.get("s3BucketName"); - // def s3Uri = "https://${bucket}.${s3_endpoint}/regression/datalake/pipeline_data/tvf/${all_type_file}"; - - // // hdfs config - // String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort") - // String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") - // def hdfsUserName = "doris" - // def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}" - // def hdfsUri = "${defaultFS}" + "/user/doris/preinstalled_data/avro/avro_all_types/${all_type_file}" - - // // TVF s3() - // qt_1 """ - // desc function s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "FORMAT" = "${format}"); - // """ - - // qt_2 """ - // select count(*) from s3( - // "uri" ="${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_1 """ - // select * from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_2 """ - // select anArray from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_3 """ - // select aMap from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_4 """ - // select anEnum from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_5 """ - // select aRecord from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_6 """ - // select aUnion from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_7 """ - // select mapArrayLong from s3( - // "uri" ="${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_8 """ - // select arrayMapBoolean from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_10 """ - // select arrayMapBoolean,aBoolean,aMap,aLong,aUnion from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_11 """ - // select aString,aDouble,anEnum from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // order_qt_12 """ - // select aRecord,aMap,aFloat from s3( - // "uri" = "${s3Uri}", - // "ACCESS_KEY" = "${ak}", - // "SECRET_KEY" = "${sk}", - // "REGION" = "${region}", - // "provider" = "${getS3Provider()}", - // "FORMAT" = "${format}"); - // """ - - // // TVF hdfs() - // String enabled = context.config.otherConfigs.get("enableHiveTest") - // if (enabled != null && enabled.equalsIgnoreCase("true")) { - // try { - // qt_3 """ - // desc function HDFS( - // "uri" = "${hdfsUri}", - // "fs.defaultFS" = "${defaultFS}", - // "hadoop.username" = "${hdfsUserName}", - // "FORMAT" = "${format}"); """ - - // order_qt_9 """ select * from HDFS( - // "uri" = "${hdfsUri}", - // "fs.defaultFS" = "${defaultFS}", - // "hadoop.username" = "${hdfsUserName}", - // "format" = "${format}")""" - - // qt_4 """ select count(*) from HDFS( - // "uri" = "${hdfsUri}", - // "fs.defaultFS" = "${defaultFS}", - // "hadoop.username" = "${hdfsUserName}", - // "format" = "${format}"); """ - - // order_qt_13 """ select arrayMapBoolean,aBoolean,aMap,aLong,aUnion from HDFS( - // "uri" = "${hdfsUri}", - // "fs.defaultFS" = "${defaultFS}", - // "hadoop.username" = "${hdfsUserName}", - // "format" = "${format}")""" - - // order_qt_14 """ select aString,aDouble,anEnum from HDFS( - // "uri" = "${hdfsUri}", - // "fs.defaultFS" = "${defaultFS}", - // "hadoop.username" = "${hdfsUserName}", - // "format" = "${format}")""" - - // order_qt_15 """ select aRecord,aMap,aFloat from HDFS( - // "uri" = "${hdfsUri}", - // "fs.defaultFS" = "${defaultFS}", - // "hadoop.username" = "${hdfsUserName}", - // "format" = "${format}")""" - // } finally { - // } - // } -} diff --git a/regression-test/suites/external_table_p2/iceberg/test_iceberg_v3_row_lineage_large_stability.groovy b/regression-test/suites/external_table_p2/iceberg/test_iceberg_v3_row_lineage_large_stability.groovy new file mode 100644 index 00000000000000..fb753eb16e7943 --- /dev/null +++ b/regression-test/suites/external_table_p2/iceberg/test_iceberg_v3_row_lineage_large_stability.groovy @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_v3_row_lineage_large_stability", "p2,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + + String catalogName = "test_iceberg_v3_row_lineage_large_stability" + String dbName = "test_row_lineage_large_stability_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + + int rowCount = (context.config.otherConfigs.get("icebergRowLineageLargeRows") ?: "500000").toInteger() + def formats = ["parquet", "orc"] + def dataColumnNames = (1..498).collect { idx -> String.format("c%03d", idx) } + + def assertChecksum = { tableName, expectedRows -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg(""" + select count(*), sum(id), sum(c001) + from demo.${dbName}.${tableName} + """) + def dorisRows = sql(""" + select count(*), sum(id), sum(c001) + from ${tableName} + """) + log.info("Spark checksum for ${tableName}: ${sparkRows}") + log.info("Doris checksum for ${tableName}: ${dorisRows}") + assertSparkDorisResultEquals(sparkRows, dorisRows) + assertEquals(expectedRows, dorisRows[0][0].toString().toLong()) + assertTrue(dorisRows[0][1] != null, "sum(id) should be non-null for ${tableName}") + assertTrue(dorisRows[0][2] != null, "sum(c001) should be non-null for ${tableName}") + return dorisRows[0].collect { it == null ? null : it.toString() } + } + + def assertPuffinDeleteFiles = { tableName -> + def deleteFiles = sql(""" + select file_path, lower(file_format), record_count + from ${tableName}\$delete_files + order by file_path + """) + log.info("Large stability delete files for ${tableName}: ${deleteFiles}") + assertTrue(deleteFiles.size() > 0, "Large stability DML should create delete files for ${tableName}") + deleteFiles.each { row -> + assertEquals("puffin", row[1].toString()) + assertTrue(row[0].toString().toLowerCase().endsWith(".puffin")) + } + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + try { + formats.each { format -> + String tableName = "large_stability_${format}" + try { + sql """drop table if exists ${tableName}""" + String dataColumns = dataColumnNames.collect { col -> "${col} int" }.join(",\n ") + String dataSelectExprs = dataColumnNames.withIndex().collect { col, idx -> + "cast(number % ${1000 + idx} as int) as ${col}" + }.join(",\n ") + sql """ + create table ${tableName} ( + id int, + dt date, + ${dataColumns} + ) engine=iceberg + partition by list (day(dt)) () + properties ( + "format-version" = "3", + "write.format.default" = "${format}", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + + sql """ + insert into ${tableName} + select + cast(number as int) as id, + cast(case when number % 2 = 0 then '2024-05-01' else '2024-05-02' end as date) as dt, + ${dataSelectExprs} + from numbers("number" = "${rowCount}") + """ + + sql """ + update ${tableName} + set c001 = c001 + 1 + where dt = date '2024-05-01' and id % 10 = 0 + """ + sql """ + delete from ${tableName} + where dt = date '2024-05-02' and id % 20 = 1 + """ + + long deletedRows = rowCount <= 1 ? 0L : (((rowCount - 2) / 20).toLong() + 1L) + long expectedRows = rowCount - deletedRows + def beforeRewriteChecksum = assertChecksum(tableName, expectedRows) + assertPuffinDeleteFiles(tableName) + + def rewriteResult = sql(""" + alter table ${catalogName}.${dbName}.${tableName} + execute rewrite_data_files( + "target-file-size-bytes" = "536870912", + "min-input-files" = "1" + ) + """) + log.info("Large stability rewrite result for ${tableName}: ${rewriteResult}") + assertTrue(rewriteResult.size() > 0, + "rewrite_data_files should return summary rows for ${tableName}") + + def afterRewriteChecksum = assertChecksum(tableName, expectedRows) + assertEquals(beforeRewriteChecksum, afterRewriteChecksum) + + def lineageCounts = sql(""" + select count(*), count(_row_id), count(_last_updated_sequence_number) + from ${tableName} + """) + log.info("Large stability lineage counts for ${tableName}: ${lineageCounts}") + assertEquals(expectedRows, lineageCounts[0][0].toString().toLong()) + assertEquals(expectedRows, lineageCounts[0][1].toString().toLong()) + assertEquals(expectedRows, lineageCounts[0][2].toString().toLong()) + } finally { + sql """drop table if exists ${tableName}""" + } + } + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p2/iceberg/test_iceberg_v3_row_lineage_random_cross_engine.groovy b/regression-test/suites/external_table_p2/iceberg/test_iceberg_v3_row_lineage_random_cross_engine.groovy new file mode 100644 index 00000000000000..992b306e09d4f1 --- /dev/null +++ b/regression-test/suites/external_table_p2/iceberg/test_iceberg_v3_row_lineage_random_cross_engine.groovy @@ -0,0 +1,270 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.Random + +suite("test_iceberg_v3_row_lineage_random_cross_engine", "p2,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("Iceberg test is disabled") + return + } + + String catalogName = "test_iceberg_v3_row_lineage_random_cross_engine" + String dbName = "test_row_lineage_random_cross_db" + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String endpoint = "http://${externalEnvIp}:${minioPort}" + + int rounds = (context.config.otherConfigs.get("icebergRowLineageRandomRounds") ?: "20").toInteger() + long seed = (context.config.otherConfigs.get("icebergRowLineageRandomSeed") ?: "61398").toLong() + def formats = ["parquet", "orc"] + + def hasSparkIcebergJdbc = { + try { + spark_iceberg """select 1""" + return true + } catch (Exception e) { + logger.info("Check spark-iceberg JDBC failed: ${e.message}") + return false + } + } + + def dorisChecksum = { tableName -> + def rows = sql(""" + select count(*), coalesce(sum(id), 0), coalesce(sum(score), 0) + from ${tableName} + """) + return rows[0].collect { it.toString() }.join(",") + } + + def sparkChecksum = { tableName -> + def rows = spark_iceberg(""" + select concat_ws(',', cast(count(*) as string), + cast(coalesce(sum(id), 0) as string), + cast(coalesce(sum(score), 0) as string)) + from demo.${dbName}.${tableName} + """) + assertEquals(1, rows.size()) + return rows[0][0].toString() + } + + def assertDorisSparkBusinessEqual = { tableName, step -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + String doris = dorisChecksum(tableName) + String spark = sparkChecksum(tableName) + logger.info("Random cross step ${step} checksum for ${tableName}: Doris=${doris}, Spark=${spark}") + assertEquals(doris, spark) + } + + def assertLineageReadable = { tableName, step -> + def rows = sql(""" + select count(*), count(_row_id), count(_last_updated_sequence_number) + from ${tableName} + """) + logger.info("Random cross step ${step} lineage counts for ${tableName}: ${rows}") + assertEquals(rows[0][0].toString().toLong(), rows[0][1].toString().toLong()) + assertEquals(rows[0][0].toString().toLong(), rows[0][2].toString().toLong()) + } + + def logSnapshots = { tableName, step -> + def snapshots = sql(""" + select snapshot_id, operation + from ${tableName}\$snapshots + order by committed_at + """) + logger.info("Random cross step ${step} snapshots for ${tableName}: ${snapshots}") + } + + def refreshBothEngines = { tableName -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + } + + def rowValues = { id, prefix, score -> + int monthDay = (id % 9) + 1 + return "${id}, '${prefix}_${id}', ${score}, date '2024-10-${String.format('%02d', monthDay)}'" + } + + def executeDorisStep = { tableName, op, id, step -> + String sqlText + if (op == 0) { + sqlText = "insert into ${tableName} values (${rowValues(id, "doris_i_${step}", id * 10 + step)})" + } else if (op == 1) { + sqlText = "update ${tableName} set name = concat(name, '_du${step}'), score = score + 1 where id = ${id}" + } else if (op == 2) { + sqlText = "delete from ${tableName} where id = ${id}" + } else if (op == 3) { + int updateId = id + int deleteId = ((id + 1) % 30) + 1 + int insertId = 10000 + step + sqlText = """ + merge into ${tableName} t + using ( + select ${updateId} as id, 'doris_mu_${step}' as name, ${updateId * 10 + step} as score, + date '2024-10-${String.format('%02d', (updateId % 9) + 1)}' as dt, 'U' as flag + union all + select ${deleteId}, 'unused', 0, + date '2024-10-${String.format('%02d', (deleteId % 9) + 1)}', 'D' + union all + select ${insertId}, 'doris_mi_${step}', ${insertId + step}, + date '2024-10-${String.format('%02d', (insertId % 9) + 1)}', 'I' + ) s + on t.id = s.id + when matched and s.flag = 'D' then delete + when matched then update set name = s.name, score = s.score + when not matched then insert (id, name, score, dt) + values (s.id, s.name, s.score, s.dt) + """ + } else { + sqlText = """ + alter table ${catalogName}.${dbName}.${tableName} + execute rewrite_data_files( + "target-file-size-bytes" = "10485760", + "min-input-files" = "1" + ) + """ + } + logger.info("Random cross step ${step} Doris SQL for ${tableName}: ${sqlText}") + sql sqlText + } + + def executeSparkStep = { tableName, op, id, step -> + String sqlText + if (op == 0) { + sqlText = "insert into demo.${dbName}.${tableName} values (${rowValues(id, "spark_i_${step}", id * 10 + step)})" + } else if (op == 1) { + sqlText = "update demo.${dbName}.${tableName} set name = concat(name, '_su${step}'), score = score + 1 where id = ${id}" + } else if (op == 2) { + sqlText = "delete from demo.${dbName}.${tableName} where id = ${id}" + } else if (op == 3) { + int updateId = id + int deleteId = ((id + 1) % 30) + 1 + int insertId = 20000 + step + sqlText = """ + merge into demo.${dbName}.${tableName} t + using ( + select ${updateId} as id, 'spark_mu_${step}' as name, ${updateId * 10 + step} as score, + date '2024-10-${String.format('%02d', (updateId % 9) + 1)}' as dt, 'U' as flag + union all + select ${deleteId}, 'unused', 0, + date '2024-10-${String.format('%02d', (deleteId % 9) + 1)}', 'D' + union all + select ${insertId}, 'spark_mi_${step}', ${insertId + step}, + date '2024-10-${String.format('%02d', (insertId % 9) + 1)}', 'I' + ) s + on t.id = s.id + when matched and s.flag = 'D' then delete + when matched then update set name = s.name, score = s.score + when not matched then insert (id, name, score, dt) + values (s.id, s.name, s.score, s.dt) + """ + } else { + sqlText = """ + call demo.system.rewrite_data_files( + table => '${dbName}.${tableName}', + options => map('target-file-size-bytes', '10485760', 'min-input-files', '1') + ) + """ + } + logger.info("Random cross step ${step} Spark SQL for ${tableName}: ${sqlText}") + spark_iceberg sqlText + } + + if (!hasSparkIcebergJdbc()) { + logger.info("spark-iceberg JDBC is unavailable, skip random cross-engine branch") + return + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog if not exists ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "${endpoint}", + "s3.region" = "us-east-1" + ) + """ + + sql """switch ${catalogName}""" + sql """create database if not exists ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + sql """set show_hidden_columns = false""" + + try { + spark_iceberg """create database if not exists demo.${dbName}""" + + formats.each { format -> + String tableName = "random_cross_${format}" + Random random = new Random(seed + format.hashCode()) + try { + spark_iceberg_multi """ + drop table if exists demo.${dbName}.${tableName}; + create table demo.${dbName}.${tableName} ( + id int, + name string, + score int, + dt date + ) using iceberg + partitioned by (days(dt)) + tblproperties ( + 'format-version' = '3', + 'write.format.default' = '${format}', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ); + insert into demo.${dbName}.${tableName} values + (1, 'base_1', 10, date '2024-10-01'), + (2, 'base_2', 20, date '2024-10-02'), + (3, 'base_3', 30, date '2024-10-03'), + (4, 'base_4', 40, date '2024-10-04'), + (5, 'base_5', 50, date '2024-10-05'); + """ + refreshBothEngines(tableName) + assertDorisSparkBusinessEqual(tableName, "initial") + assertLineageReadable(tableName, "initial") + + for (int step = 1; step <= rounds; step++) { + boolean useDoris = random.nextBoolean() + int op = random.nextInt(5) + int id = random.nextInt(30) + 1 + if (useDoris) { + executeDorisStep(tableName, op, id, step) + } else { + executeSparkStep(tableName, op, id, step) + } + refreshBothEngines(tableName) + assertDorisSparkBusinessEqual(tableName, step) + assertLineageReadable(tableName, step) + logSnapshots(tableName, step) + } + } finally { + sql """drop table if exists ${tableName}""" + } + } + } finally { + sql """drop database if exists ${dbName} force""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy b/regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy index 7ebc54cd60bc11..13aadd908ab406 100644 --- a/regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy @@ -32,8 +32,8 @@ suite("test_oceanbase_jdbc_catalog", "p2,external") { sql """ create catalog if not exists ${catalog_name} properties( "type"="jdbc", "user"="root@test", - "password"="", - "jdbc_url" = "jdbc:oceanbase://${externalEnvIp}:${oceanbase_port}/doris_test", + "password"="123456", + "jdbc_url" = "jdbc:oceanbase://${externalEnvIp}:${oceanbase_port}/doris_test?serverTimezone=UTC", "driver_url" = "${driver_url}", "driver_class" = "com.oceanbase.jdbc.Driver" );""" diff --git a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy index 5a7e4584cdb94f..fd2337bd8d86da 100644 --- a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy +++ b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy @@ -122,6 +122,9 @@ suite("test_paimon_hms_catalog", "p2,external") { // kerberos String hdfs_kerberos_properties = """ "fs.defaultFS" = "hdfs://${extHiveHmsHost}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab" @@ -129,6 +132,9 @@ suite("test_paimon_hms_catalog", "p2,external") { String hdfs_new_kerberos_properties = """ "fs.defaultFS" = "hdfs://${extHiveHmsHost}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hdfs.authentication.type" = "kerberos", "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hdfs.authentication.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab" @@ -226,4 +232,3 @@ suite("test_paimon_hms_catalog", "p2,external") { testQuery(paimon_hms_catalog_properties + paimon_fs_gcs_support + gcs_warehouse_properties + gcs_storage_s3_properties, "support_gcs_s3", "gcs_db") } - diff --git a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy index 86a25d07d0d46a..87d276b4d61993 100644 --- a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy +++ b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy @@ -481,6 +481,9 @@ suite("hive_on_hms_and_dlf", "p2,external") { // kerberos String hdfs_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -488,6 +491,9 @@ suite("hive_on_hms_and_dlf", "p2,external") { """ String hdfs_new_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hdfs.authentication.type" = "kerberos", "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hdfs.authentication.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab" @@ -541,8 +547,6 @@ suite("hive_on_hms_and_dlf", "p2,external") { String db_location = "obs://${obs_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + obs_storage_properties, "hive_hms_obs_test", db_location) testQueryAndInsert(hms_properties + obs_region_param + obs_storage_properties, "hive_hms_obs_test_region", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + obs_storage_properties, "hive_hms_on_obs_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + obs_storage_properties, "hive_hms_on_obs_kerberos_new", db_location) //OBS - Partition table tests db_location = "obs://${obs_parent_path}/hive/hms/partition/" + System.currentTimeMillis() @@ -555,8 +559,6 @@ suite("hive_on_hms_and_dlf", "p2,external") { db_location = "gs://${gcs_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + gcs_storage_old_properties, "hive_hms_gcs_test", db_location) testQueryAndInsert(hms_properties + gcs_storage_new_properties, "hive_hms_gcs_test_new", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + gcs_storage_old_properties, "hive_hms_on_gcs_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + gcs_storage_new_properties, "hive_hms_on_gcs_kerberos_new", db_location) //GCS - Insert overwrite tests db_location = "gs://${gcs_parent_path}/hive/hms/overwrite/" + System.currentTimeMillis() testInsertOverwrite(hms_properties + gcs_storage_new_properties, "hive_hms_gcs_overwrite_test", db_location) @@ -566,8 +568,6 @@ suite("hive_on_hms_and_dlf", "p2,external") { db_location = "cosn://${cos_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + cos_storage_properties, "hive_hms_cos_test", db_location) testQueryAndInsert(hms_properties + cos_region_param + cos_storage_properties, "hive_hms_cos_test_region", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + cos_storage_properties, "hive_hms_on_cos_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + cos_storage_properties, "hive_hms_on_cos_kerberos_new", db_location) //COS - Partition table tests db_location = "cosn://${cos_parent_path}/hive/hms/partition/" + System.currentTimeMillis() @@ -583,8 +583,6 @@ suite("hive_on_hms_and_dlf", "p2,external") { db_location = "oss://${oss_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + oss_storage_properties, "hive_hms_oss_test", db_location) testQueryAndInsert(hms_properties + oss_region_param + oss_storage_properties, "hive_hms_oss_test_region", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + oss_storage_properties, "hive_hms_on_oss_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + oss_storage_properties, "hive_hms_on_oss_kerberos_new", db_location) //OSS - Partition table tests (fix for partition path scheme mismatch) db_location = "oss://${oss_parent_path}/hive/hms/partition/" + System.currentTimeMillis() diff --git a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy index 45215c76d123fb..a9d73886259663 100644 --- a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy +++ b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy @@ -477,6 +477,9 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external") { // kerberos String hdfs_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.security.authentication" = "kerberos", "io-impl" = "org.apache.doris.datasource.iceberg.fileio.DelegateFileIO", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -485,6 +488,9 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external") { String hdfs_new_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "io-impl" = "org.apache.doris.datasource.iceberg.fileio.DelegateFileIO", "hdfs.authentication.type" = "kerberos", "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -543,10 +549,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external") { hmsTestQueryAndInsert(hms_prop + warehouse + oss_region_param + oss_storage_properties, "iceberg_hms_on_oss") - //old kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_old") - //new kerberos - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_new") warehouse = """ 'warehouse' = 'oss://${oss_bucket_endpoint_parent_path}/iceberg-hms-warehouse', """ @@ -560,10 +562,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external") { + warehouse + obs_storage_properties, "iceberg_hms_on_obs") hmsTestQueryAndInsert(hms_prop+ warehouse + obs_region_param + obs_storage_properties, "iceberg_hms_on_obs") - //old kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_old") - //new kerberos - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_new") /*--------HMS on GCS-----------*/ if(context.config.otherConfigs.get("enableGCS")){ @@ -575,10 +573,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external") { hmsTestQueryAndInsert(hms_prop+ warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_new") - //new kerberos - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_new") - //old kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_old") } /*--------HMS on COS-----------*/ @@ -593,9 +587,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external") { 'warehouse' = 'cos://${cos_parent_path}/iceberg-hms-cos-warehouse', """ hmsTestQueryAndInsert(hms_prop + warehouse + cos_storage_properties, "iceberg_hms_on_cos") - //kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + cos_storage_properties, "iceberg_hms_on_cos_kerberos_old") - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + cos_storage_properties, "iceberg_hms_on_cos_kerberos_new") /*--------HMS on S3-----------*/ diff --git a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_publish_conflict_seq.groovy b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_publish_conflict_seq.groovy index 8e9d692b35d58e..4aa2d660be7095 100644 --- a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_publish_conflict_seq.groovy +++ b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_publish_conflict_seq.groovy @@ -32,6 +32,7 @@ suite("test_partial_update_publish_conflict_seq", "nonConcurrent") { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", diff --git a/regression-test/suites/fault_injection_p0/test_audit_log_internal_query_failure.groovy b/regression-test/suites/fault_injection_p0/test_audit_log_internal_query_failure.groovy deleted file mode 100644 index 233187e582e1dd..00000000000000 --- a/regression-test/suites/fault_injection_p0/test_audit_log_internal_query_failure.groovy +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Regression test for CIR-20019: when an internal query fails (for example -// the column-statistics gathering SQL that ANALYZE issues against a user -// table), the audit log entry must record state=ERR with a descriptive -// error_message instead of the previous misleading state=OK / return_rows=0. -suite('test_audit_log_internal_query_failure', 'nonConcurrent') { - def tbl = 'test_audit_log_internal_query_failure_t1' - - setGlobalVarTemporary([enable_audit_plugin: true], { - try { - sql "drop table if exists ${tbl}" - sql """ - create table ${tbl} (k int, v int) - duplicate key(k) - distributed by hash(k) buckets 1 - properties('replication_num'='1') - """ - sql "insert into ${tbl} values(1,10),(2,20),(3,30)" - - // Limit the injected IO error to this table's tablet so concurrent - // reads on system tables (such as audit_log itself) are unaffected. - def tabletRows = sql_return_maparray "show tablets from ${tbl}" - assertFalse(tabletRows.isEmpty(), "expected at least one tablet for ${tbl}") - def tabletId = tabletRows[0].TabletId - - // Capture the FE-side current timestamp to filter audit rows so - // stale entries from previous runs do not satisfy the assertion. - def startTime = (sql_return_maparray "select now() as ts")[0].ts.toString() - - GetDebugPoint().clearDebugPointsForAllBEs() - try { - GetDebugPoint().enableDebugPointForAllBEs( - "LocalFileReader::read_at_impl.io_error", - [ sub_path: "/${tabletId}/" ]) - - test { - sql "analyze table ${tbl} with sync" - exception "IO_ERROR" - } - } finally { - GetDebugPoint().clearDebugPointsForAllBEs() - } - - def currentDb = (sql_return_maparray "select database() as db")[0].db.toString() - def fullTableName = "internal.${currentDb}.${tbl}" - - // Force a flush so the failed internal query is queryable from - // __internal_schema.audit_log. - // The failed gather SQL reads from our user table and runs as an - // internal query; it must show up with state=ERR. Filter by start - // time to avoid matching stale entries from previous runs. - // Use queried_tables_and_views instead of stmt because stmt can be - // truncated by audit_plugin_max_sql_length when running in CI. - def query = """select state, error_code, error_message - from __internal_schema.audit_log - where is_internal = 1 - and array_contains(queried_tables_and_views, '${fullTableName}') - and state = 'ERR' - and `time` >= '${startTime}' - order by `time` desc limit 1""" - def res = [] - int retry = 60 - while (res.isEmpty() && retry-- > 0) { - sql "call flush_audit_log()" - sleep(2000) - res = sql_return_maparray "${query}" - } - assertFalse(res.isEmpty(), - "expected an audit_log entry with state=ERR for the failed gather query") - assertEquals('ERR', res[0].state.toString()) - assertNotEquals('0', res[0].error_code.toString()) - assertNotNull(res[0].error_message) - assertTrue(!res[0].error_message.toString().isEmpty(), - "audit_log error_message should not be empty, got: ${res[0].error_message}") - } finally { - try { - sql "drop table if exists ${tbl}" - } catch (Throwable ignored) { - } - } - }) -} diff --git a/regression-test/suites/fault_injection_p0/test_incomplete_commit_info.groovy b/regression-test/suites/fault_injection_p0/test_incomplete_commit_info.groovy index 2b6503a929e3fa..7394fa9ac5c792 100644 --- a/regression-test/suites/fault_injection_p0/test_incomplete_commit_info.groovy +++ b/regression-test/suites/fault_injection_p0/test_incomplete_commit_info.groovy @@ -41,7 +41,23 @@ suite("test_incomplete_commit_info", "nonConcurrent") { ) engine=olap DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1") """ - GetDebugPoint().enableDebugPointForAllBEs("VNodeChannel.add_block_success_callback.incomplete_commit_info") + + streamLoad { + table "${tableName}" + db "regression_test_fault_injection_p0" + set 'column_separator', ',' + file "baseall.txt" + } + + def tabletIds = sql_return_maparray("SHOW TABLETS FROM ${tableName}") + .collect { it.TabletId } + .unique() + def tabletId = tabletIds.find { + sql("SELECT COUNT(*) FROM ${tableName} TABLET(${it})")[0][0].toLong() > 0 + } + GetDebugPoint().enableDebugPointForAllBEs( + "VNodeChannel.add_block_success_callback.incomplete_commit_info", + [tablet_id: "${tabletId}"]) streamLoad { table "${tableName}" db "regression_test_fault_injection_p0" @@ -55,9 +71,11 @@ suite("test_incomplete_commit_info", "nonConcurrent") { log.info("Stream load result: ${result}".toString()) def json = parseJson(result) assertEquals("fail", json.Status.toLowerCase()) + assertTrue(json.Message.contains("Failed to commit txn")) + assertTrue(json.Message.contains("succ replica num 0 < load required replica num 1")) } } } finally { GetDebugPoint().disableDebugPointForAllBEs("VNodeChannel.add_block_success_callback.incomplete_commit_info") } -} \ No newline at end of file +} diff --git a/regression-test/suites/function_p0/cast/test_cast_null_array_to_nested_array.groovy b/regression-test/suites/function_p0/cast/test_cast_null_array_to_nested_array.groovy new file mode 100644 index 00000000000000..a777a90eff7033 --- /dev/null +++ b/regression-test/suites/function_p0/cast/test_cast_null_array_to_nested_array.groovy @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_cast_null_array_to_nested_array") { + sql "set debug_skip_fold_constant = false;" + + qt_cast_empty_array_to_nested_array_fold """ + select cast([] as array>); + """ + + qt_cast_null_array_to_nested_array_fold """ + select cast([null] as array>); + """ + + qt_cast_nulls_array_to_nested_array_fold """ + select cast([null, null] as array>); + """ + + qt_cast_nested_null_array_to_deeper_array_fold """ + select cast([[null]] as array>>); + """ + + qt_cast_deeply_nested_null_array_to_deeper_array_fold """ + select cast([[[null]]] as array>>>); + """ + + qt_cast_nested_null_array_to_same_dimension_fold """ + select cast([[null]] as array>); + """ + + test { + sql "select cast([[null]] as array);" + exception "can not cast from origin type" + } + + test { + sql "select cast([[[null]]] as array>);" + exception "can not cast from origin type" + } + + test { + sql "select cast([[1]] as array>>);" + exception "can not cast from origin type" + } + + test { + sql "select cast([1] as array>);" + exception "can not cast from origin type ARRAY to target type=ARRAY>" + } + + test { + sql "select cast([1, null] as array>);" + exception "can not cast from origin type ARRAY to target type=ARRAY>" + } + + sql "set debug_skip_fold_constant = true;" + + qt_cast_empty_array_to_nested_array_be """ + select cast([] as array>); + """ + + qt_cast_null_array_to_nested_array_be """ + select cast([null] as array>); + """ + + qt_cast_nulls_array_to_nested_array_be """ + select cast([null, null] as array>); + """ + + qt_cast_nested_null_array_to_deeper_array_be """ + select cast([[null]] as array>>); + """ + + qt_cast_deeply_nested_null_array_to_deeper_array_be """ + select cast([[[null]]] as array>>>); + """ + + qt_cast_nested_null_array_to_same_dimension_be """ + select cast([[null]] as array>); + """ + + test { + sql "select cast([[null]] as array);" + exception "can not cast from origin type" + } + + test { + sql "select cast([[[null]]] as array>);" + exception "can not cast from origin type" + } + + test { + sql "select cast([[1]] as array>>);" + exception "can not cast from origin type" + } + + test { + sql "select cast([1] as array>);" + exception "can not cast from origin type ARRAY to target type=ARRAY>" + } + + test { + sql "select cast([1, null] as array>);" + exception "can not cast from origin type ARRAY to target type=ARRAY>" + } +} diff --git a/regression-test/suites/insert_p0/group_commit/test_group_commit_wal_num_backpressure.groovy b/regression-test/suites/insert_p0/group_commit/test_group_commit_wal_num_backpressure.groovy new file mode 100644 index 00000000000000..478fc2c71566d1 --- /dev/null +++ b/regression-test/suites/insert_p0/group_commit/test_group_commit_wal_num_backpressure.groovy @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.awaitility.Awaitility +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_group_commit_wal_num_backpressure", "nonConcurrent") { + def getRowCount = { expectedRowCount -> + Awaitility.await().atMost(60, SECONDS).pollInterval(1, SECONDS).until( + { + def result = sql "select count(*) from test_group_commit_wal_num_backpressure" + logger.info("table: test_group_commit_wal_num_backpressure, rowCount: ${result}, expectedRowCount: ${expectedRowCount}") + return result[0][0] == expectedRowCount + } + ) + } + + sql """ DROP TABLE IF EXISTS test_group_commit_wal_num_backpressure """ + sql """ + CREATE TABLE IF NOT EXISTS test_group_commit_wal_num_backpressure ( + `k` int, + `v` int + ) engine=olap + DISTRIBUTED BY HASH(`k`) + BUCKETS 1 + properties( + "replication_num" = "1", + "group_commit_interval_ms" = "10000", + "group_commit_data_bytes" = "1" + ) + """ + + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + def rowCount = 0 + try { + setBeConfigTemporary([group_commit_max_wal_num_per_table: 5]) { + GetDebugPoint().enableDebugPointForAllBEs("LoadBlockQueue._finish_group_commit_load.load_error") + GetDebugPoint().enableDebugPointForAllBEs("WalTable::_handle_stream_load.fail") + + def backendIps = [:] + def backendHttpPorts = [:] + getBackendIpHttpPort(backendIps, backendHttpPorts) + def backendId = backendIps.keySet()[0] + def beHost = backendIps.get(backendId) + def beHttpPort = backendHttpPorts.get(backendId) as int + + def streamLoadToBe = { + streamLoad { + table "test_group_commit_wal_num_backpressure" + set 'column_separator', ',' + set 'group_commit', 'async_mode' + unset 'label' + file 'group_commit_wal_msg.csv' + time 10000 + directToBe beHost, beHttpPort + } + rowCount += 5 + } + + def blocked = false + def maxAttempts = 100 + for (int i = 0; i < maxAttempts && !blocked; ++i) { + try { + streamLoadToBe() + sleep(i < 10 ? 100 : 1000) + } catch (Exception e) { + logger.info("catch expected exception: " + e.getMessage()) + assertTrue(e.getMessage().contains("Too many group commit async WALs")) + assertTrue(e.getMessage().contains("limit=5")) + assertTrue(e.getMessage().contains("last replay wal failed reason")) + assertTrue(e.getMessage().contains("WalTable::_handle_stream_load.fail")) + blocked = true + break + } + } + assertTrue(blocked) + } + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + } + + // getRowCount(rowCount) +} diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy index 02aebbc3ecb258..3dacfe71a652f7 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy @@ -93,8 +93,8 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { def process = cmd.execute() def output = new StringBuffer() def errorOutput = new StringBuffer() - process.consumeProcessOutput(output, errorOutput) - int exitCode = process.waitFor() + process.waitForProcessOutput(output, errorOutput) + int exitCode = process.exitValue() logger.info("Show config: code=" + exitCode + ", out=" + output + ", err=" + errorOutput) assertEquals(exitCode, 0) def configList = parseJson(output.toString().trim()) diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.groovy index 1955476bee154c..0b5753ac9309b7 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.groovy @@ -90,9 +90,9 @@ suite("test_streaming_mysql_job_all_type", "p0,external,mysql,external_docker,ex `enum` enum('Value1', 'Value2', 'Value3') ) engine=innodb charset=utf8; """ - // mock snapshot data + // Use numeric YEAR 0 to verify yearIsDateType=false preserves 0000 instead of 2000. sql """ - insert into ${mysqlDb}.${table1} values (1,120,50000,1000000,9000000000,1000,12345.67,987654.12345,123456789.1234567890,99999999.123456789012345678901234567890,123.45,12.34,true,-5,-300,-20000,-500000,-8000000000,-123.45,-12.34,-1000,-1234.56,-98765.43210,-123456789.1234567890,-99999999.123456789012345678901234567890,2023,'08:30:00','08:30:00.123','08:30:00.123456','2023-06-15','2023-06-15 08:30:00','2023-06-15 08:30:00','2023-06-15 08:30:00.123','2023-06-15 08:30:00.123456','abc','user_001','normal text content','simple blob data','{"id":1,"name":"userA"}','Option1',b'110011','binary_data1','varbin_01','Value1'); + insert into ${mysqlDb}.${table1} values (1,120,50000,1000000,9000000000,1000,12345.67,987654.12345,123456789.1234567890,99999999.123456789012345678901234567890,123.45,12.34,true,-5,-300,-20000,-500000,-8000000000,-123.45,-12.34,-1000,-1234.56,-98765.43210,-123456789.1234567890,-99999999.123456789012345678901234567890,0,'08:30:00','08:30:00.123','08:30:00.123456','2023-06-15','2023-06-15 08:30:00','2023-06-15 08:30:00','2023-06-15 08:30:00.123','2023-06-15 08:30:00.123456','abc','user_001','normal text content','simple blob data','{"id":1,"name":"userA"}','Option1',b'110011','binary_data1','varbin_01','Value1'); """ } diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_charset_and_strings.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_charset_and_strings.groovy index dcec811ed2729b..f89e937f1a6fd6 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_charset_and_strings.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_charset_and_strings.groovy @@ -31,8 +31,8 @@ import static java.util.concurrent.TimeUnit.SECONDS // Same themes run twice: ids 1-6 via JDBC/snapshot, ids 101-106 via binlog. // Plus UPDATEs that switch a row's payload through binlog. // -// JDBC URL uses characterEncoding=utf8 to keep the driver round-trip stable; -// gbk and latin1 columns are converted by MySQL on insert. +// The streaming job URL omits Unicode and character encoding options to verify +// that the MySQL defaults are added before metadata discovery and snapshot reads. suite("test_streaming_mysql_job_charset_and_strings", "p0,external,mysql,external_docker,external_docker_mysql,nondatalake") { def jobName = "test_streaming_mysql_job_charset_and_strings_name" def currentDb = (sql "select database()")[0][0] @@ -98,7 +98,7 @@ suite("test_streaming_mysql_job_charset_and_strings", "p0,external,mysql,externa sql """CREATE JOB ${jobName} ON STREAMING FROM MYSQL ( - "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}?serverTimezone=UTC&useUnicode=true&characterEncoding=utf8", + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}?serverTimezone=UTC", "driver_url" = "${driver_url}", "driver_class" = "com.mysql.cj.jdbc.Driver", "user" = "root", diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_create_alter.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_create_alter.groovy index a0dbd5e36cea6e..2e6443db633583 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_create_alter.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_create_alter.groovy @@ -334,18 +334,9 @@ suite("test_streaming_mysql_job_create_alter", "p0,external,mysql,external_docke test { sql """ALTER JOB ${jobName} FROM MYSQL ( - "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}", - "driver_url" = "${driver_url}", - "driver_class" = "com.mysql.cj.jdbc.Driver", - "user" = "root", - "password" = "123456", - "database" = "${mysqlDb}", - "include_tables" = "changeTable", - "offset" = "initial" - ) - TO DATABASE ${currentDb} ( - "table.create.properties.replication_num" = "1" + "include_tables" = "changeTable" ) + TO DATABASE ${currentDb} """ exception "The include_tables property cannot be modified in ALTER JOB" } @@ -354,19 +345,9 @@ suite("test_streaming_mysql_job_create_alter", "p0,external,mysql,external_docke test { sql """ALTER JOB ${jobName} FROM MYSQL ( - "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}", - "driver_url" = "${driver_url}", - "driver_class" = "com.mysql.cj.jdbc.Driver", - "user" = "root", - "password" = "123456", - "database" = "${mysqlDb}", - "include_tables" = "${table1}", - "exclude_tables" = "xxxx", - "offset" = "initial" - ) - TO DATABASE ${currentDb} ( - "table.create.properties.replication_num" = "1" + "exclude_tables" = "xxxx" ) + TO DATABASE ${currentDb} """ exception "The exclude_tables property cannot be modified in ALTER JOB" } @@ -433,15 +414,7 @@ suite("test_streaming_mysql_job_create_alter", "p0,external,mysql,external_docke test { sql """ALTER JOB ${jobName} FROM MYSQL ( - "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}", - "driver_url" = "${driver_url}", - "driver_class" = "com.mysql.cj.jdbc.Driver", - "user" = "root", - "password" = "123456", - "database" = "${mysqlDb}", - "include_tables" = "${table1}", - "offset" = "initial", - "xxx"="xxx" + "xxx" = "xxx" ) TO DATABASE ${currentDb} """ diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_integer_boundary.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_integer_boundary.groovy index be00ffa73893f8..80e29d18da0b0b 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_integer_boundary.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_integer_boundary.groovy @@ -22,7 +22,7 @@ import static java.util.concurrent.TimeUnit.SECONDS // Guard MySQL integer-type boundaries that all_type does NOT cover: // 1) UNSIGNED boundaries (esp. BIGINT UNSIGNED >= 2^63, which overflows Java long) -// 2) TINYINT(1) <-> BOOLEAN mapping (MySQL ambiguity not present in PG) +// 2) TINYINT(1) remains TINYINT and preserves non-boolean values // // Coverage is run twice: ids 1-5 cover the snapshot path, ids 101-105 repeat // the same boundary themes through the binlog path. Plus UPDATEs that switch @@ -60,20 +60,20 @@ suite("test_streaming_mysql_job_integer_boundary", "p0,external,mysql,external_d `mediumint_u` mediumint unsigned, `int_u` int unsigned, `bigint_u` bigint unsigned, - `bool_col` tinyint(1), + `tinyint_1` tinyint(1), `tinyint_s` tinyint ) engine=innodb default charset=utf8; """ // ----- Snapshot rows: 5 boundary themes via JDBC path ----- - // tinyint_u smallint_u mediumint_u int_u bigint_u bool tinyint_s - sql """insert into ${mysqlDb}.${table1} values (1, 'all_zero', 0, 0, 0, 0, 0, 0, 0)""" - sql """insert into ${mysqlDb}.${table1} values (2, 'signed_max', 127, 32767, 8388607, 2147483647, 9223372036854775807, 1, 127)""" + // tinyint_u smallint_u mediumint_u int_u bigint_u tinyint(1) tinyint_s + sql """insert into ${mysqlDb}.${table1} values (1, 'all_zero', 0, 0, 0, 0, 0, 0, 0)""" + sql """insert into ${mysqlDb}.${table1} values (2, 'signed_max', 127, 32767, 8388607, 2147483647, 9223372036854775807, 1, 127)""" // signed_max+1: every column passes its signed boundary; bigint_u steps into Java long overflow range. - sql """insert into ${mysqlDb}.${table1} values (3, 'signed_max_plus1', 128, 32768, 8388608, 2147483648, 9223372036854775808, 0, -128)""" + sql """insert into ${mysqlDb}.${table1} values (3, 'signed_max_plus1', 128, 32768, 8388608, 2147483648, 9223372036854775808, -1, -128)""" // unsigned_max: each column at its unsigned upper bound. bigint_u is 2^64-1. - sql """insert into ${mysqlDb}.${table1} values (4, 'unsigned_max', 255, 65535, 16777215, 4294967295, 18446744073709551615, 1, -1)""" - sql """insert into ${mysqlDb}.${table1} values (5, 'mid_value', 100, 30000, 8000000, 2000000000, 9000000000000000000, 0, 50)""" + sql """insert into ${mysqlDb}.${table1} values (4, 'unsigned_max', 255, 65535, 16777215, 4294967295, 18446744073709551615, 127, -1)""" + sql """insert into ${mysqlDb}.${table1} values (5, 'mid_value', 100, 30000, 8000000, 2000000000, 9000000000000000000, 4, 50)""" } sql """CREATE JOB ${jobName} @@ -113,18 +113,18 @@ suite("test_streaming_mysql_job_integer_boundary", "p0,external,mysql,external_d // Verify type mapping in Doris (tinyint_u->smallint, bigint_u->largeint, etc.) qt_desc_integer_boundary """desc ${currentDb}.${table1};""" - qt_select_snapshot """select id, tag, tinyint_u, smallint_u, mediumint_u, int_u, bigint_u, bool_col, tinyint_s from ${currentDb}.${table1} order by id;""" + qt_select_snapshot """select id, tag, tinyint_u, smallint_u, mediumint_u, int_u, bigint_u, tinyint_1, tinyint_s from ${currentDb}.${table1} order by id;""" // ===== Binlog phase: repeat the SAME 5 boundary themes through binlog path ===== connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}") { - sql """insert into ${mysqlDb}.${table1} values (101, 'all_zero', 0, 0, 0, 0, 0, 0, 0)""" - sql """insert into ${mysqlDb}.${table1} values (102, 'signed_max', 127, 32767, 8388607, 2147483647, 9223372036854775807, 1, 127)""" - sql """insert into ${mysqlDb}.${table1} values (103, 'signed_max_plus1', 128, 32768, 8388608, 2147483648, 9223372036854775808, 0, -128)""" - sql """insert into ${mysqlDb}.${table1} values (104, 'unsigned_max', 255, 65535, 16777215, 4294967295, 18446744073709551615, 1, -1)""" - sql """insert into ${mysqlDb}.${table1} values (105, 'mid_value', 100, 30000, 8000000, 2000000000, 9000000000000000000, 0, 50)""" + sql """insert into ${mysqlDb}.${table1} values (101, 'all_zero', 0, 0, 0, 0, 0, 0, 0)""" + sql """insert into ${mysqlDb}.${table1} values (102, 'signed_max', 127, 32767, 8388607, 2147483647, 9223372036854775807, 1, 127)""" + sql """insert into ${mysqlDb}.${table1} values (103, 'signed_max_plus1', 128, 32768, 8388608, 2147483648, 9223372036854775808, -1, -128)""" + sql """insert into ${mysqlDb}.${table1} values (104, 'unsigned_max', 255, 65535, 16777215, 4294967295, 18446744073709551615, 127, -1)""" + sql """insert into ${mysqlDb}.${table1} values (105, 'mid_value', 100, 30000, 8000000, 2000000000, 9000000000000000000, 4, 50)""" sql """update ${mysqlDb}.${table1} set bigint_u=18446744073709551615 where id=1""" - sql """update ${mysqlDb}.${table1} set bool_col=0 where id=2""" + sql """update ${mysqlDb}.${table1} set tinyint_1=4 where id=2""" sql """update ${mysqlDb}.${table1} set int_u=4294967295 where id=5""" } @@ -135,15 +135,16 @@ suite("test_streaming_mysql_job_integer_boundary", "p0,external,mysql,external_d { def cnt = sql """select count(1) from ${currentDb}.${table1}""" def upd1 = sql """select cast(bigint_u as string) from ${currentDb}.${table1} where id=1""" - def upd2 = sql """select bool_col from ${currentDb}.${table1} where id=2""" + def upd2 = sql """select tinyint_1 from ${currentDb}.${table1} where id=2""" def upd5 = sql """select int_u from ${currentDb}.${table1} where id=5""" def b1 = upd1.get(0).get(0) == null ? '' : upd1.get(0).get(0).toString() - def b2 = upd2.get(0).get(0) + def t2 = upd2.get(0).get(0) def b5 = upd5.get(0).get(0) - log.info("incr count=" + cnt + " id1.bigint_u=" + b1 + " id2.bool_col=" + b2 + " id5.int_u=" + b5) + log.info("incr count=" + cnt + " id1.bigint_u=" + b1 + + " id2.tinyint_1=" + t2 + " id5.int_u=" + b5) cnt.get(0).get(0) == 10 && b1 == '18446744073709551615' && - b2 != null && b2.toString() == 'false' && + t2 != null && t2.toString() == '4' && b5 != null && b5.toString() == '4294967295' } ) @@ -155,7 +156,7 @@ suite("test_streaming_mysql_job_integer_boundary", "p0,external,mysql,external_d throw ex } - qt_select_after_incr """select id, tag, tinyint_u, smallint_u, mediumint_u, int_u, bigint_u, bool_col, tinyint_s from ${currentDb}.${table1} order by id;""" + qt_select_after_incr """select id, tag, tinyint_u, smallint_u, mediumint_u, int_u, bigint_u, tinyint_1, tinyint_s from ${currentDb}.${table1} order by id;""" sql """DROP JOB IF EXISTS where jobname = '${jobName}'""" diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_with_concurrent_dml.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_with_concurrent_dml.groovy index 597bea37320270..b387fa5dd81d96 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_with_concurrent_dml.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_snapshot_with_concurrent_dml.groovy @@ -18,19 +18,22 @@ import org.awaitility.Awaitility +import static java.util.concurrent.TimeUnit.MILLISECONDS import static java.util.concurrent.TimeUnit.SECONDS suite("test_streaming_mysql_job_snapshot_with_concurrent_dml", "p0,external,mysql,external_docker,external_docker_mysql,nondatalake") { def jobName = "test_streaming_mysql_job_snapshot_with_concurrent_dml_name" def currentDb = (sql "select database()")[0][0] def table1 = "streaming_snapshot_dml_mysql" - def unrelated = "streaming_snapshot_dml_unrelated_mysql" + def table2 = "streaming_snapshot_dml_other_pk_mysql" + def excludedTable = "streaming_snapshot_dml_excluded_mysql" def mysqlDb = "test_cdc_db" def totalRows = 1000 sql """DROP JOB IF EXISTS where jobname = '${jobName}'""" sql """drop table if exists ${currentDb}.${table1} force""" - sql """drop table if exists ${currentDb}.${unrelated} force""" + sql """drop table if exists ${currentDb}.${table2} force""" + sql """drop table if exists ${currentDb}.${excludedTable} force""" String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { @@ -59,17 +62,33 @@ suite("test_streaming_mysql_job_snapshot_with_concurrent_dml", "p0,external,mysq } sql sb.toString() - sql """DROP TABLE IF EXISTS ${mysqlDb}.${unrelated}""" - sql """CREATE TABLE ${mysqlDb}.${unrelated} ( + sql """DROP TABLE IF EXISTS ${mysqlDb}.${table2}""" + sql """CREATE TABLE ${mysqlDb}.${table2} ( + `other_id` int NOT NULL, + `tag` varchar(64), + `version` int, + PRIMARY KEY (`other_id`) + ) ENGINE=InnoDB""" + + StringBuilder table2Rows = new StringBuilder() + table2Rows.append("INSERT INTO ${mysqlDb}.${table2} (other_id, tag, version) VALUES ") + for (int i = 1; i <= totalRows; i++) { + if (i > 1) table2Rows.append(", ") + table2Rows.append("(${i}, 'snap', 0)") + } + sql table2Rows.toString() + + sql """DROP TABLE IF EXISTS ${mysqlDb}.${excludedTable}""" + sql """CREATE TABLE ${mysqlDb}.${excludedTable} ( `id` int NOT NULL, `tag` varchar(64), PRIMARY KEY (`id`) ) ENGINE=InnoDB""" - sql """INSERT INTO ${mysqlDb}.${unrelated} (id, tag) VALUES (1, 'pre_snap')""" + sql """INSERT INTO ${mysqlDb}.${excludedTable} (id, tag) VALUES (1, 'pre_snap')""" } - // snapshot_split_size=10 + snapshot_parallelism=1 -> 100 serial splits, slow enough that - // the concurrent DML below actually overlaps with snapshot. + // Two tables with different primary-key names make a foreign-table backfill record + // incompatible with the current split key. Serial small splits keep the snapshot active. sql """CREATE JOB ${jobName} ON STREAMING FROM MYSQL ( @@ -79,27 +98,54 @@ suite("test_streaming_mysql_job_snapshot_with_concurrent_dml", "p0,external,mysq "user" = "root", "password" = "123456", "database" = "${mysqlDb}", - "include_tables" = "${table1}", + "include_tables" = "${table1},${table2}", "offset" = "initial", "snapshot_split_size" = "10", - "snapshot_parallelism" = "1" + "snapshot_parallelism" = "1", + "skip_snapshot_backfill" = "false" ) TO DATABASE ${currentDb} ( "table.create.properties.replication_num" = "1" ) """ - // Concurrent DML on source while cdc-client is still snapshotting. + def succeedTaskCount = { + def rows = sql """select SucceedTaskCount from jobs("type"="insert") where Name='${jobName}' and ExecuteType='STREAMING'""" + rows.size() == 1 ? (rows.get(0).get(0).toString() as long) : 0L + } + Awaitility.await().atMost(120, SECONDS).pollInterval(100, MILLISECONDS).until({ + succeedTaskCount() >= 1 + }) + + def writeProbeDml = { + connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}") { + sql """UPDATE ${mysqlDb}.${table1} SET version=version+1 WHERE id=500""" + sql """UPDATE ${mysqlDb}.${table2} SET version=version+1 WHERE other_id=500""" + } + } + + writeProbeDml() + long probeStartTaskCount = succeedTaskCount() + long probeTargetTaskCount = probeStartTaskCount + 10 + Awaitility.await().atMost(120, SECONDS).pollInterval(200, MILLISECONDS).until({ + writeProbeDml() + succeedTaskCount() >= probeTargetTaskCount + }) + log.info("MySQL probe DML covered tasks ${probeStartTaskCount}..${succeedTaskCount()}") + + // Apply deterministic DML after the probe updates so final results remain stable. connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}") { for (int i = 1; i <= 10; i++) { sql """INSERT INTO ${mysqlDb}.${table1} (id, tag, version) VALUES (${totalRows + i}, 'concurrent_ins', 1)""" + sql """INSERT INTO ${mysqlDb}.${table2} (other_id, tag, version) VALUES (${totalRows + i}, 'concurrent_ins', 1)""" } sql """UPDATE ${mysqlDb}.${table1} SET version=99 WHERE id IN (1, 100, 500, 999)""" sql """DELETE FROM ${mysqlDb}.${table1} WHERE id IN (2, 200, 800)""" + sql """UPDATE ${mysqlDb}.${table2} SET version=99 WHERE other_id IN (1, 100, 500, 999)""" + sql """DELETE FROM ${mysqlDb}.${table2} WHERE other_id IN (2, 200, 800)""" - // DML on unrelated table - must NOT leak into Doris (not in include_tables). - sql """INSERT INTO ${mysqlDb}.${unrelated} (id, tag) VALUES (2, 'concurrent_unrelated_ins')""" - sql """UPDATE ${mysqlDb}.${unrelated} SET tag='concurrent_unrelated_upd' WHERE id=1""" + sql """INSERT INTO ${mysqlDb}.${excludedTable} (id, tag) VALUES (2, 'concurrent_excluded_ins')""" + sql """UPDATE ${mysqlDb}.${excludedTable} SET tag='concurrent_excluded_upd' WHERE id=1""" } def expectedRows = totalRows + 10 - 3 @@ -113,15 +159,26 @@ suite("test_streaming_mysql_job_snapshot_with_concurrent_dml", "p0,external,mysq def del2 = sql """select count(1) from ${currentDb}.${table1} where id=2""" def del800 = sql """select count(1) from ${currentDb}.${table1} where id=800""" def ins1010 = sql """select count(1) from ${currentDb}.${table1} where id=${totalRows + 10}""" + def cnt2 = sql """select count(1) from ${currentDb}.${table2}""" + def upd2 = sql """select version from ${currentDb}.${table2} where other_id=999""" + def del2Table2 = sql """select count(1) from ${currentDb}.${table2} where other_id=2""" + def ins2 = sql """select count(1) from ${currentDb}.${table2} where other_id=${totalRows + 10}""" def v1 = upd1.size() == 0 ? null : upd1.get(0).get(0) def v999 = upd999.size() == 0 ? null : upd999.get(0).get(0) - log.info("incr cnt=${cnt} v1=${v1} v999=${v999} del2=${del2} del800=${del800} ins1010=${ins1010}") + def v2 = upd2.size() == 0 ? null : upd2.get(0).get(0) + log.info("incr cnt=${cnt} cnt2=${cnt2} v1=${v1} v999=${v999} v2=${v2} " + + "del2=${del2} del800=${del800} del2Table2=${del2Table2} " + + "ins1010=${ins1010} ins2=${ins2}") cnt.get(0).get(0) == expectedRows && + cnt2.get(0).get(0) == expectedRows && v1 != null && v1.toString() == '99' && v999 != null && v999.toString() == '99' && + v2 != null && v2.toString() == '99' && del2.get(0).get(0) == 0 && del800.get(0).get(0) == 0 && - ins1010.get(0).get(0) == 1 + del2Table2.get(0).get(0) == 0 && + ins1010.get(0).get(0) == 1 && + ins2.get(0).get(0) == 1 } ) } catch (Exception ex) { @@ -132,13 +189,17 @@ suite("test_streaming_mysql_job_snapshot_with_concurrent_dml", "p0,external,mysq throw ex } - def showUnrelated = sql """show tables from ${currentDb} like '${unrelated}'""" - assert showUnrelated.size() == 0 + def showExcluded = sql """show tables from ${currentDb} like '${excludedTable}'""" + assert showExcluded.size() == 0 qt_select_count """select count(1) from ${currentDb}.${table1}""" + qt_select_count_t2 """select count(1) from ${currentDb}.${table2}""" qt_select_updates """select id, version from ${currentDb}.${table1} where id in (1, 100, 500, 999) order by id""" + qt_select_updates_t2 """select other_id, version from ${currentDb}.${table2} where other_id in (1, 100, 500, 999) order by other_id""" qt_select_deletes """select count(1) from ${currentDb}.${table1} where id in (2, 200, 800)""" + qt_select_deletes_t2 """select count(1) from ${currentDb}.${table2} where other_id in (2, 200, 800)""" qt_select_inserts """select id, tag, version from ${currentDb}.${table1} where id > ${totalRows} order by id""" + qt_select_inserts_t2 """select other_id, tag, version from ${currentDb}.${table2} where other_id > ${totalRows} order by other_id""" sql """DROP JOB IF EXISTS where jobname = '${jobName}'""" diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_special_offset.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_special_offset.groovy index 3c988239225a65..c4234913661623 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_special_offset.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_special_offset.groovy @@ -182,18 +182,9 @@ suite("test_streaming_mysql_job_special_offset", "p0,external,mysql,external_doc test { sql """ALTER JOB ${jobName} FROM MYSQL ( - "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}", - "driver_url" = "${driver_url}", - "driver_class" = "com.mysql.cj.jdbc.Driver", - "user" = "root", - "password" = "123456", - "database" = "${mysqlDb}", - "include_tables" = "${table1}", "offset" = "latest" ) - TO DATABASE ${currentDb} ( - "table.create.properties.replication_num" = "1" - ) + TO DATABASE ${currentDb} """ exception "The offset in source properties cannot be modified in ALTER JOB" } diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job.groovy new file mode 100644 index 00000000000000..416c8ddc01646e --- /dev/null +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job.groovy @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_streaming_oceanbase_job", + "p2,external,oceanbase,external_docker,external_docker_oceanbase,nondatalake") { + def jobName = "test_streaming_oceanbase_job" + def currentDb = (sql "SELECT DATABASE()")[0][0] + def sourceDb = "test_oceanbase_streaming_db" + def table1 = "oceanbase_streaming_users" + def table2 = "oceanbase_streaming_orders" + def emptyTable = "oceanbase_streaming_empty" + + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + sql """DROP TABLE IF EXISTS ${currentDb}.${table1} FORCE""" + sql """DROP TABLE IF EXISTS ${currentDb}.${table2} FORCE""" + sql """DROP TABLE IF EXISTS ${currentDb}.${emptyTable} FORCE""" + + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String oceanbaseCdcPort = context.config.otherConfigs.get("oceanbase_cdc_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driverUrl = + "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + + def dumpJobState = { + log.info("jobs: " + sql("""SELECT * FROM jobs("type"="insert") WHERE Name='${jobName}'""")) + log.info("tasks: " + sql("""SELECT * FROM tasks("type"="insert") WHERE JobName='${jobName}'""")) + } + + connect("root@test", "123456", "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}") { + sql """CREATE DATABASE IF NOT EXISTS ${sourceDb}""" + sql """DROP TABLE IF EXISTS ${sourceDb}.${table1}""" + sql """DROP TABLE IF EXISTS ${sourceDb}.${table2}""" + sql """DROP TABLE IF EXISTS ${sourceDb}.${emptyTable}""" + sql """CREATE TABLE ${sourceDb}.${table1} ( + id INT NOT NULL, + name VARCHAR(100), + age INT, + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + sql """CREATE TABLE ${sourceDb}.${table2} ( + id INT NOT NULL, + description VARCHAR(100), + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + sql """CREATE TABLE ${sourceDb}.${emptyTable} ( + id INT NOT NULL, + value VARCHAR(100), + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + sql """INSERT INTO ${sourceDb}.${table1} VALUES + (1, 'Alice', 18), + (2, 'Bob', 20)""" + sql """INSERT INTO ${sourceDb}.${table2} VALUES (10, 'snapshot_order')""" + } + + sql """CREATE JOB ${jobName} + ON STREAMING + FROM OCEANBASE ( + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root@test", + "password" = "123456", + "database" = "${sourceDb}", + "include_tables" = "${table1},${table2},${emptyTable}", + "offset" = "initial" + ) + TO DATABASE ${currentDb} ( + "table.create.properties.replication_num" = "1" + )""" + + try { + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def users = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}""" + def orders = sql """SELECT COUNT(*) FROM ${currentDb}.${table2}""" + def emptyTables = sql """SHOW TABLES FROM ${currentDb} LIKE '${emptyTable}'""" + users[0][0] == 2 && orders[0][0] == 1 && emptyTables.size() == 1 + }) + } catch (Exception ex) { + dumpJobState() + throw ex + } + + order_qt_oceanbase_snapshot_users """ + SELECT id, name, age FROM ${currentDb}.${table1} ORDER BY id + """ + order_qt_oceanbase_snapshot_orders """ + SELECT id, description FROM ${currentDb}.${table2} ORDER BY id + """ + order_qt_oceanbase_snapshot_empty """ + SELECT id, value FROM ${currentDb}.${emptyTable} ORDER BY id + """ + + connect("root@test", "123456", "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}") { + sql """INSERT INTO ${sourceDb}.${table1} VALUES (3, 'Carol', 30)""" + sql """UPDATE ${sourceDb}.${table1} SET age=21 WHERE id=2""" + sql """DELETE FROM ${sourceDb}.${table1} WHERE id=1""" + sql """INSERT INTO ${sourceDb}.${table2} VALUES (11, 'incremental_order')""" + } + + try { + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def users = sql """SELECT id, name, age FROM ${currentDb}.${table1} ORDER BY id""" + def orders = sql """SELECT id, description FROM ${currentDb}.${table2} ORDER BY id""" + users == [[2, 'Bob', 21], [3, 'Carol', 30]] && + orders == [[10, 'snapshot_order'], [11, 'incremental_order']] + }) + } catch (Exception ex) { + dumpJobState() + throw ex + } + + order_qt_oceanbase_incremental_users """ + SELECT id, name, age FROM ${currentDb}.${table1} ORDER BY id + """ + order_qt_oceanbase_incremental_orders """ + SELECT id, description FROM ${currentDb}.${table2} ORDER BY id + """ + + def status = sql """SELECT Status FROM jobs("type"="insert") WHERE Name='${jobName}'""" + assert status.size() == 1 && status[0][0] == "RUNNING" + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + } +} diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_all_type.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_all_type.groovy new file mode 100644 index 00000000000000..5d81bb0c2bc741 --- /dev/null +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_all_type.groovy @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_streaming_oceanbase_job_all_type", + "p2,external,oceanbase,external_docker,external_docker_oceanbase,nondatalake") { + def jobName = "test_streaming_oceanbase_job_all_type" + def currentDb = (sql "SELECT DATABASE()")[0][0] + def sourceDb = "test_oceanbase_streaming_db" + def table1 = "oceanbase_streaming_all_type" + + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + sql """DROP TABLE IF EXISTS ${currentDb}.${table1} FORCE""" + + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String oceanbaseCdcPort = context.config.otherConfigs.get("oceanbase_cdc_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driverUrl = + "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + String sourceUrl = "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}?serverTimezone=UTC" + + def dumpJobState = { + log.info("jobs: " + sql("""SELECT * FROM jobs("type"="insert") WHERE Name='${jobName}'""")) + log.info("tasks: " + sql("""SELECT * FROM tasks("type"="insert") WHERE JobName='${jobName}'""")) + } + + connect("root@test", "123456", sourceUrl) { + sql """CREATE DATABASE IF NOT EXISTS ${sourceDb}""" + sql """DROP TABLE IF EXISTS ${sourceDb}.${table1}""" + sql """CREATE TABLE ${sourceDb}.${table1} ( + id INT NOT NULL, + tiny_unsigned TINYINT UNSIGNED, + big_unsigned BIGINT UNSIGNED, + amount DECIMAL(18, 5), + enabled BOOLEAN, + event_date DATE, + event_datetime DATETIME(6), + event_timestamp TIMESTAMP(6) NULL, + fixed_text CHAR(5), + variable_text VARCHAR(32), + long_text TEXT, + binary_value BLOB, + json_value JSON, + set_value SET('A', 'B', 'C'), + enum_value ENUM('NEW', 'DONE'), + varbinary_value VARBINARY(16), + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + sql """INSERT INTO ${sourceDb}.${table1} VALUES ( + 1, 200, 18446744073709551610, 123456789.12345, TRUE, + '2026-07-10', '2026-07-10 10:11:12.123456', + '2026-07-10 10:11:12.123456', 'abc', 'snapshot', + 'snapshot text', X'010203', '{"id":1,"name":"snapshot"}', + 'A,C', 'NEW', X'0A0B0C' + )""" + sql """INSERT INTO ${sourceDb}.${table1} VALUES ( + 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL + )""" + } + + sql """CREATE JOB ${jobName} + ON STREAMING + FROM OCEANBASE ( + "jdbc_url" = "${sourceUrl}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root@test", + "password" = "123456", + "database" = "${sourceDb}", + "include_tables" = "${table1}", + "offset" = "initial" + ) + TO DATABASE ${currentDb} ( + "table.create.properties.replication_num" = "1" + )""" + + try { + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}""" + rows.size() == 1 && rows[0][0] == 2 + }) + } catch (Exception ex) { + dumpJobState() + throw ex + } + + order_qt_oceanbase_all_type_snapshot """ + SELECT id, tiny_unsigned, big_unsigned, amount, enabled, event_date, + event_datetime, event_timestamp, fixed_text, variable_text, + long_text, HEX(binary_value), CAST(json_value AS STRING), + set_value, enum_value, HEX(varbinary_value) + FROM ${currentDb}.${table1} + ORDER BY id + """ + + connect("root@test", "123456", sourceUrl) { + sql """INSERT INTO ${sourceDb}.${table1} VALUES ( + 3, 100, 9000000000, -98765.43210, FALSE, + '2026-07-11', '2026-07-11 11:12:13.654321', + '2026-07-11 11:12:13.654321', 'xyz', 'incremental', + 'incremental text', X'FFEEDD', '{"id":3,"name":"incremental"}', + 'B', 'DONE', X'ABCDEF' + )""" + sql """UPDATE ${sourceDb}.${table1} + SET variable_text='updated', amount=1.25000 WHERE id=1""" + sql """DELETE FROM ${sourceDb}.${table1} WHERE id=2""" + } + + try { + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def count = sql """SELECT COUNT(*) FROM ${currentDb}.${table1}""" + def updated = sql """SELECT variable_text FROM ${currentDb}.${table1} WHERE id=1""" + def inserted = sql """SELECT COUNT(*) FROM ${currentDb}.${table1} WHERE id=3""" + count[0][0] == 2 && updated.size() == 1 && updated[0][0] == 'updated' && + inserted[0][0] == 1 + }) + } catch (Exception ex) { + dumpJobState() + throw ex + } + + order_qt_oceanbase_all_type_incremental """ + SELECT id, tiny_unsigned, big_unsigned, amount, enabled, event_date, + event_datetime, event_timestamp, fixed_text, variable_text, + long_text, HEX(binary_value), CAST(json_value AS STRING), + set_value, enum_value, HEX(varbinary_value) + FROM ${currentDb}.${table1} + ORDER BY id + """ + + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + } +} diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_offset.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_offset.groovy new file mode 100644 index 00000000000000..3fa947c87c5cfb --- /dev/null +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_offset.groovy @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_streaming_oceanbase_job_offset", + "p2,external,oceanbase,external_docker,external_docker_oceanbase,nondatalake") { + def currentDb = (sql "SELECT DATABASE()")[0][0] + def sourceDb = "test_oceanbase_streaming_db" + def suffix = UUID.randomUUID().toString().replace("-", "").substring(0, 8) + def latestJob = "test_streaming_oceanbase_offset_latest" + def earliestJob = "test_streaming_oceanbase_offset_earliest" + def specificJob = "test_streaming_oceanbase_offset_specific" + def latestTable = "oceanbase_offset_latest_${suffix}" + def earliestTable = "oceanbase_offset_earliest_${suffix}" + def specificTable = "oceanbase_offset_specific_${suffix}" + + [latestJob, earliestJob, specificJob].each { + sql """DROP JOB IF EXISTS WHERE jobname='${it}'""" + } + [latestTable, earliestTable, specificTable].each { + sql """DROP TABLE IF EXISTS ${currentDb}.${it} FORCE""" + } + + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String oceanbaseCdcPort = context.config.otherConfigs.get("oceanbase_cdc_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driverUrl = + "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + String sourceUrl = "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}" + + def waitForJobReady = { String jobName -> + Awaitility.await().atMost(180, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql """SELECT Status, SucceedTaskCount FROM jobs("type"="insert") + WHERE Name='${jobName}'""" + rows.size() == 1 && rows[0][0] == "RUNNING" && (rows[0][1] as int) >= 1 + }) + } + def waitForRows = { String tableName, int count -> + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql """SELECT COUNT(*) FROM ${currentDb}.${tableName}""" + rows.size() == 1 && rows[0][0] == count + }) + } + def waitForRowId = { String tableName, int id -> + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql """SELECT COUNT(*) FROM ${currentDb}.${tableName} WHERE id = ${id}""" + rows.size() == 1 && rows[0][0] == 1 + }) + } + def dumpJobState = { String jobName -> + log.info("jobs: " + sql("""SELECT * FROM jobs("type"="insert") WHERE Name='${jobName}'""")) + log.info("tasks: " + sql("""SELECT * FROM tasks("type"="insert") WHERE JobName='${jobName}'""")) + } + + connect("root@test", "123456", sourceUrl) { + sql """CREATE DATABASE IF NOT EXISTS ${sourceDb}""" + sql """DROP TABLE IF EXISTS ${sourceDb}.${latestTable}""" + sql """CREATE TABLE ${sourceDb}.${latestTable} ( + id INT NOT NULL, + name VARCHAR(100), + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + sql """INSERT INTO ${sourceDb}.${latestTable} VALUES (1, 'before_latest')""" + } + + // Consume later records first to ensure before_latest is visible in OBBinlog. + def binlogFile = "" + def binlogPosition = "" + connect("root@test", "123456", sourceUrl) { + sql """DROP TABLE IF EXISTS ${sourceDb}.${specificTable}""" + sql """CREATE TABLE ${sourceDb}.${specificTable} ( + id INT NOT NULL, + name VARCHAR(100), + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + def masterStatus = sql """SHOW MASTER STATUS""" + binlogFile = masterStatus[0][0] + binlogPosition = masterStatus[0][1].toString() + sql """INSERT INTO ${sourceDb}.${specificTable} VALUES + (10, 'specific_one'), + (11, 'specific_two')""" + } + def offsetJson = """{"file":"${binlogFile}","pos":"${binlogPosition}"}""" + + sql """CREATE JOB ${specificJob} + ON STREAMING + FROM OCEANBASE ( + "jdbc_url" = "${sourceUrl}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root@test", + "password" = "123456", + "database" = "${sourceDb}", + "include_tables" = "${specificTable}", + "offset" = '${offsetJson}' + ) + TO DATABASE ${currentDb} ( + "table.create.properties.replication_num" = "1" + )""" + + try { + waitForRows(specificTable, 2) + } catch (Exception ex) { + dumpJobState(specificJob) + throw ex + } + sql """DROP JOB IF EXISTS WHERE jobname='${specificJob}'""" + + sql """CREATE JOB ${latestJob} + ON STREAMING + FROM OCEANBASE ( + "jdbc_url" = "${sourceUrl}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root@test", + "password" = "123456", + "database" = "${sourceDb}", + "include_tables" = "${latestTable}", + "offset" = "latest" + ) + TO DATABASE ${currentDb} ( + "table.create.properties.replication_num" = "1" + )""" + + try { + waitForJobReady(latestJob) + connect("root@test", "123456", sourceUrl) { + sql """INSERT INTO ${sourceDb}.${latestTable} VALUES (2, 'after_latest')""" + } + waitForRowId(latestTable, 2) + } catch (Exception ex) { + dumpJobState(latestJob) + throw ex + } + + order_qt_oceanbase_offset_latest """ + SELECT id, name FROM ${currentDb}.${latestTable} ORDER BY id + """ + sql """DROP JOB IF EXISTS WHERE jobname='${latestJob}'""" + + connect("root@test", "123456", sourceUrl) { + sql """DROP TABLE IF EXISTS ${sourceDb}.${earliestTable}""" + sql """CREATE TABLE ${sourceDb}.${earliestTable} ( + id INT NOT NULL, + name VARCHAR(100), + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + sql """INSERT INTO ${sourceDb}.${earliestTable} VALUES + (1, 'earliest_one'), + (2, 'earliest_two')""" + } + + sql """CREATE JOB ${earliestJob} + ON STREAMING + FROM OCEANBASE ( + "jdbc_url" = "${sourceUrl}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root@test", + "password" = "123456", + "database" = "${sourceDb}", + "include_tables" = "${earliestTable}", + "offset" = "earliest" + ) + TO DATABASE ${currentDb} ( + "table.create.properties.replication_num" = "1" + )""" + + try { + waitForRows(earliestTable, 2) + } catch (Exception ex) { + dumpJobState(earliestJob) + throw ex + } + + order_qt_oceanbase_offset_earliest """ + SELECT id, name FROM ${currentDb}.${earliestTable} ORDER BY id + """ + sql """DROP JOB IF EXISTS WHERE jobname='${earliestJob}'""" + + order_qt_oceanbase_offset_specific """ + SELECT id, name FROM ${currentDb}.${specificTable} ORDER BY id + """ + } +} diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_restart_fe.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_restart_fe.groovy new file mode 100644 index 00000000000000..d61732a31f9f71 --- /dev/null +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_restart_fe.groovy @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_streaming_oceanbase_job_restart_fe", + "p2,docker,oceanbase,external_docker,external_docker_oceanbase,nondatalake") { + def jobName = "test_streaming_oceanbase_job_restart_fe" + def sourceDb = "test_oceanbase_streaming_db" + def table1 = "oceanbase_streaming_restart_fe" + def options = new ClusterOptions() + options.setFeNum(1) + options.cloudMode = null + + docker(options) { + def currentDb = (sql "SELECT DATABASE()")[0][0] + + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + sql """DROP TABLE IF EXISTS ${currentDb}.${table1} FORCE""" + + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String oceanbaseCdcPort = context.config.otherConfigs.get("oceanbase_cdc_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driverUrl = + "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + String sourceUrl = "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}" + + def waitForValue = { int id, String expected -> + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql "SELECT value FROM ${currentDb}.${table1} WHERE id=${id}" + rows.size() == 1 && rows[0][0] == expected + }) + } + def waitForJobAfterRestart = { + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS) + .ignoreExceptions().until({ + context.reconnectFe() + def rows = sql """SELECT Status FROM jobs("type"="insert") + WHERE Name='${jobName}'""" + rows.size() == 1 && rows[0][0] == "RUNNING" + }) + } + def dumpJobState = { + log.info("jobs: " + sql("""SELECT * FROM jobs("type"="insert") WHERE Name='${jobName}'""")) + log.info("tasks: " + sql("""SELECT * FROM tasks("type"="insert") WHERE JobName='${jobName}'""")) + } + + connect("root@test", "123456", sourceUrl) { + sql """CREATE DATABASE IF NOT EXISTS ${sourceDb}""" + sql """DROP TABLE IF EXISTS ${sourceDb}.${table1}""" + sql """CREATE TABLE ${sourceDb}.${table1} ( + id INT NOT NULL, + value VARCHAR(100), + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + sql """INSERT INTO ${sourceDb}.${table1} VALUES (1, 'snapshot')""" + } + + sql """CREATE JOB ${jobName} + ON STREAMING + FROM OCEANBASE ( + "jdbc_url" = "${sourceUrl}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root@test", + "password" = "123456", + "database" = "${sourceDb}", + "include_tables" = "${table1}", + "offset" = "initial" + ) + TO DATABASE ${currentDb} ( + "table.create.properties.replication_num" = "1" + )""" + + try { + waitForValue(1, "snapshot") + connect("root@test", "123456", sourceUrl) { + sql """INSERT INTO ${sourceDb}.${table1} VALUES (2, 'before_restart')""" + } + waitForValue(2, "before_restart") + } catch (Exception ex) { + dumpJobState() + throw ex + } + + def beforeOffset = null + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql """SELECT currentOffset FROM jobs("type"="insert") + WHERE Name='${jobName}'""" + if (rows.size() != 1 || rows[0][0] == null) { + return false + } + def parsed = parseJson(rows[0][0]) + if (parsed.file == null || parsed.pos == null) { + return false + } + beforeOffset = parsed + return true + }) + + cluster.restartFrontends() + waitForJobAfterRestart() + context.reconnectFe() + + def afterOffsetRows = sql """SELECT currentOffset FROM jobs("type"="insert") + WHERE Name='${jobName}'""" + assert afterOffsetRows.size() == 1 && afterOffsetRows[0][0] != null + def afterOffset = parseJson(afterOffsetRows[0][0]) + assert afterOffset.file > beforeOffset.file || + (afterOffset.file == beforeOffset.file && + (afterOffset.pos as long) >= (beforeOffset.pos as long)) + + connect("root@test", "123456", sourceUrl) { + sql """INSERT INTO ${sourceDb}.${table1} VALUES (3, 'after_restart')""" + sql """UPDATE ${sourceDb}.${table1} SET value='updated_after_restart' WHERE id=2""" + } + + try { + waitForValue(3, "after_restart") + waitForValue(2, "updated_after_restart") + } catch (Exception ex) { + dumpJobState() + throw ex + } + + order_qt_oceanbase_restart_fe """ + SELECT id, value FROM ${currentDb}.${table1} ORDER BY id + """ + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + } + } +} diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_sc_restart_fe.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_sc_restart_fe.groovy new file mode 100644 index 00000000000000..fb07ad59028daf --- /dev/null +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_sc_restart_fe.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_streaming_oceanbase_job_sc_restart_fe", + "p2,docker,oceanbase,external_docker,external_docker_oceanbase,nondatalake") { + def jobName = "test_streaming_oceanbase_job_sc_restart_fe" + def sourceDb = "test_oceanbase_streaming_db" + def table1 = "oceanbase_streaming_sc_restart_fe" + def options = new ClusterOptions() + options.setFeNum(1) + options.cloudMode = null + + docker(options) { + def currentDb = (sql "SELECT DATABASE()")[0][0] + + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + sql """DROP TABLE IF EXISTS ${currentDb}.${table1} FORCE""" + + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String oceanbaseCdcPort = context.config.otherConfigs.get("oceanbase_cdc_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driverUrl = + "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + String sourceUrl = "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}" + + def waitForColumn = { String column, boolean expected -> + Awaitility.await().atMost(180, SECONDS).pollInterval(2, SECONDS).until({ + (sql "DESC ${currentDb}.${table1}").any { it[0] == column } == expected + }) + } + def waitForValue = { int id, String column, String expected -> + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql "SELECT ${column} FROM ${currentDb}.${table1} WHERE id=${id}" + rows.size() == 1 && String.valueOf(rows[0][0]) == expected + }) + } + def waitForJobAfterRestart = { + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS) + .ignoreExceptions().until({ + context.reconnectFe() + def rows = sql """SELECT Status FROM jobs("type"="insert") + WHERE Name='${jobName}'""" + rows.size() == 1 && rows[0][0] == "RUNNING" + }) + } + def dumpJobState = { + log.info("jobs: " + sql("""SELECT * FROM jobs("type"="insert") WHERE Name='${jobName}'""")) + log.info("tasks: " + sql("""SELECT * FROM tasks("type"="insert") WHERE JobName='${jobName}'""")) + } + + connect("root@test", "123456", sourceUrl) { + sql """CREATE DATABASE IF NOT EXISTS ${sourceDb}""" + sql """DROP TABLE IF EXISTS ${sourceDb}.${table1}""" + sql """CREATE TABLE ${sourceDb}.${table1} ( + id INT NOT NULL, + value VARCHAR(100), + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + sql """INSERT INTO ${sourceDb}.${table1} VALUES (1, 'snapshot')""" + } + + sql """CREATE JOB ${jobName} + ON STREAMING + FROM OCEANBASE ( + "jdbc_url" = "${sourceUrl}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root@test", + "password" = "123456", + "database" = "${sourceDb}", + "include_tables" = "${table1}", + "offset" = "initial" + ) + TO DATABASE ${currentDb} ( + "table.create.properties.replication_num" = "1" + )""" + + try { + waitForValue(1, "value", "snapshot") + connect("root@test", "123456", sourceUrl) { + sql """ALTER TABLE ${sourceDb}.${table1} ADD COLUMN extra_value VARCHAR(50)""" + sql """INSERT INTO ${sourceDb}.${table1} + VALUES (2, 'before_restart', 'schema_before_restart')""" + } + waitForColumn("extra_value", true) + waitForValue(2, "extra_value", "schema_before_restart") + } catch (Exception ex) { + dumpJobState() + throw ex + } + + cluster.restartFrontends() + waitForJobAfterRestart() + context.reconnectFe() + + connect("root@test", "123456", sourceUrl) { + sql """INSERT INTO ${sourceDb}.${table1} + VALUES (3, 'after_restart', 'schema_after_restart')""" + sql """UPDATE ${sourceDb}.${table1} + SET extra_value='updated_after_restart' WHERE id=2""" + } + + try { + waitForValue(3, "extra_value", "schema_after_restart") + waitForValue(2, "extra_value", "updated_after_restart") + } catch (Exception ex) { + dumpJobState() + throw ex + } + + order_qt_oceanbase_sc_after_restart """ + SELECT id, value, extra_value FROM ${currentDb}.${table1} ORDER BY id + """ + + connect("root@test", "123456", sourceUrl) { + sql """ALTER TABLE ${sourceDb}.${table1} DROP COLUMN extra_value""" + sql """INSERT INTO ${sourceDb}.${table1} VALUES (4, 'after_drop')""" + } + + try { + waitForColumn("extra_value", false) + waitForValue(4, "value", "after_drop") + } catch (Exception ex) { + dumpJobState() + throw ex + } + + order_qt_oceanbase_sc_restart_final """ + SELECT id, value FROM ${currentDb}.${table1} ORDER BY id + """ + def status = sql """SELECT Status FROM jobs("type"="insert") WHERE Name='${jobName}'""" + assert status.size() == 1 && status[0][0] == "RUNNING" + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + } + } +} diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_schema_change.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_schema_change.groovy new file mode 100644 index 00000000000000..0535ec60792e10 --- /dev/null +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_schema_change.groovy @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_streaming_oceanbase_job_schema_change", + "p2,external,oceanbase,external_docker,external_docker_oceanbase,nondatalake") { + def jobName = "test_streaming_oceanbase_job_schema_change" + def currentDb = (sql "SELECT DATABASE()")[0][0] + def sourceDb = "test_oceanbase_streaming_db" + def table1 = "oceanbase_streaming_schema_change" + + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + sql """DROP TABLE IF EXISTS ${currentDb}.${table1} FORCE""" + + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String oceanbaseCdcPort = context.config.otherConfigs.get("oceanbase_cdc_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driverUrl = + "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + + def waitForColumn = { String column, boolean expected -> + Awaitility.await().atMost(180, SECONDS).pollInterval(2, SECONDS).until({ + (sql "DESC ${currentDb}.${table1}").any { it[0] == column } == expected + }) + } + def waitForValue = { int id, String column, String expected -> + Awaitility.await().atMost(180, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql "SELECT ${column} FROM ${currentDb}.${table1} WHERE id=${id}" + rows.size() == 1 && String.valueOf(rows[0][0]) == expected + }) + } + def dumpJobState = { + log.info("jobs: " + sql("""SELECT * FROM jobs("type"="insert") WHERE Name='${jobName}'""")) + log.info("tasks: " + sql("""SELECT * FROM tasks("type"="insert") WHERE JobName='${jobName}'""")) + } + + connect("root@test", "123456", "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}") { + sql """CREATE DATABASE IF NOT EXISTS ${sourceDb}""" + sql """DROP TABLE IF EXISTS ${sourceDb}.${table1}""" + sql """CREATE TABLE ${sourceDb}.${table1} ( + id INT NOT NULL, + name VARCHAR(100), + PRIMARY KEY (id) + ) ENGINE=InnoDB""" + sql """INSERT INTO ${sourceDb}.${table1} VALUES (1, 'snapshot')""" + } + + sql """CREATE JOB ${jobName} + ON STREAMING + FROM OCEANBASE ( + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root@test", + "password" = "123456", + "database" = "${sourceDb}", + "include_tables" = "${table1}", + "offset" = "initial" + ) + TO DATABASE ${currentDb} ( + "table.create.properties.replication_num" = "1" + )""" + + try { + waitForValue(1, "name", "snapshot") + } catch (Exception ex) { + dumpJobState() + throw ex + } + + connect("root@test", "123456", "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}") { + sql """ALTER TABLE ${sourceDb}.${table1} ADD COLUMN city VARCHAR(50)""" + sql """INSERT INTO ${sourceDb}.${table1} VALUES (2, 'after_add', 'Hangzhou')""" + } + + try { + waitForColumn("city", true) + waitForValue(2, "city", "Hangzhou") + } catch (Exception ex) { + dumpJobState() + throw ex + } + + order_qt_oceanbase_after_add """ + SELECT id, name, city FROM ${currentDb}.${table1} ORDER BY id + """ + + connect("root@test", "123456", "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}") { + sql """UPDATE ${sourceDb}.${table1} SET city='Shanghai' WHERE id=2""" + } + + try { + waitForValue(2, "city", "Shanghai") + } catch (Exception ex) { + dumpJobState() + throw ex + } + + connect("root@test", "123456", "jdbc:mysql://${externalEnvIp}:${oceanbaseCdcPort}") { + sql """ALTER TABLE ${sourceDb}.${table1} DROP COLUMN city""" + sql """INSERT INTO ${sourceDb}.${table1} VALUES (3, 'after_drop')""" + } + + try { + waitForColumn("city", false) + waitForValue(3, "name", "after_drop") + } catch (Exception ex) { + dumpJobState() + throw ex + } + + order_qt_oceanbase_after_drop """ + SELECT id, name FROM ${currentDb}.${table1} ORDER BY id + """ + + def status = sql """SELECT Status FROM jobs("type"="insert") WHERE Name='${jobName}'""" + assert status.size() == 1 && status[0][0] == "RUNNING" + sql """DROP JOB IF EXISTS WHERE jobname='${jobName}'""" + } +} diff --git a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_snapshot_with_concurrent_dml_multi_table.groovy b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_snapshot_with_concurrent_dml_multi_table.groovy index 6ecdffdea8dd5b..6dd528a3a7e2b9 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_snapshot_with_concurrent_dml_multi_table.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_snapshot_with_concurrent_dml_multi_table.groovy @@ -18,6 +18,7 @@ import org.awaitility.Awaitility +import static java.util.concurrent.TimeUnit.MILLISECONDS import static java.util.concurrent.TimeUnit.SECONDS // Multi-table from-to snapshot + concurrent DML: with >=2 tables each snapshot split flips the @@ -31,6 +32,7 @@ suite("test_streaming_postgres_job_snapshot_with_concurrent_dml_multi_table", def table1 = "streaming_snapshot_dml_multi_pg_t1" def table2 = "streaming_snapshot_dml_multi_pg_t2" def tables = [table1, table2] + def primaryKeys = [(table1): "pk", (table2): "other_id"] def pgDB = "postgres" def pgSchema = "cdc_test" def pgUser = "postgres" @@ -51,21 +53,22 @@ suite("test_streaming_postgres_job_snapshot_with_concurrent_dml_multi_table", // ===== Prepare PG side: two tables, each 1000 snapshot rows ===== connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") { tables.each { t -> + def primaryKey = primaryKeys[t] sql """DROP TABLE IF EXISTS ${pgDB}.${pgSchema}.${t}""" sql """ create table ${pgDB}.${pgSchema}.${t} ( - id integer PRIMARY KEY, + ${primaryKey} integer PRIMARY KEY, tag varchar(64), version integer ); """ - sql """INSERT INTO ${pgDB}.${pgSchema}.${t} (id, tag, version) + sql """INSERT INTO ${pgDB}.${pgSchema}.${t} (${primaryKey}, tag, version) SELECT g, 'snap', 0 FROM generate_series(1, ${totalRows}) g""" } } - // snapshot_split_size=10 + snapshot_parallelism=1 -> 100 serial splits per table, slow - // enough that the concurrent DML overlaps snapshot while the publication keeps flipping. + // Different primary-key names make a foreign-table record incompatible with the current + // split key. Serial small splits keep the snapshot active during concurrent DML. sql """CREATE JOB ${jobName} ON STREAMING FROM POSTGRES ( @@ -79,28 +82,49 @@ suite("test_streaming_postgres_job_snapshot_with_concurrent_dml_multi_table", "include_tables" = "${table1},${table2}", "offset" = "initial", "snapshot_split_size" = "10", - "snapshot_parallelism" = "1" + "snapshot_parallelism" = "1", + "skip_snapshot_backfill" = "false" ) TO DATABASE ${currentDb} ( "table.create.properties.replication_num" = "1" ) """ - // Wait until the first snapshot split commits (slot created, snapshot in progress) so the - // DML below lands inside the snapshot window and overlaps the publication flipping. - Awaitility.await().atMost(120, SECONDS).pollInterval(1, SECONDS).until({ - def c = sql """select SucceedTaskCount from jobs("type"="insert") where Name='${jobName}' and ExecuteType='STREAMING'""" - c.size() == 1 && (c.get(0).get(0).toString() as long) >= 1 + // Wait until snapshot processing starts, then keep writing while subsequent splits run. + def succeedTaskCount = { + def rows = sql """select SucceedTaskCount from jobs("type"="insert") where Name='${jobName}' and ExecuteType='STREAMING'""" + rows.size() == 1 ? (rows.get(0).get(0).toString() as long) : 0L + } + Awaitility.await().atMost(120, SECONDS).pollInterval(100, MILLISECONDS).until({ + succeedTaskCount() >= 1 + }) + + def writeProbeDml = { + connect("${pgUser}", "${pgPassword}", + "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") { + sql """UPDATE ${pgDB}.${pgSchema}.${table1} SET version=version+1 WHERE pk=500""" + sql """UPDATE ${pgDB}.${pgSchema}.${table2} SET version=version+1 WHERE other_id=500""" + } + } + + writeProbeDml() + long probeStartTaskCount = succeedTaskCount() + long probeTargetTaskCount = probeStartTaskCount + 10 + Awaitility.await().atMost(120, SECONDS).pollInterval(200, MILLISECONDS).until({ + writeProbeDml() + succeedTaskCount() >= probeTargetTaskCount }) + log.info("PostgreSQL probe DML covered tasks ${probeStartTaskCount}..${succeedTaskCount()}") - // Concurrent DML on BOTH tables while still snapshotting. Same DML shape on each table. + // Apply deterministic DML after the probe updates so final results remain stable. connect("${pgUser}", "${pgPassword}", "jdbc:postgresql://${externalEnvIp}:${pg_port}/${pgDB}") { tables.each { t -> + def primaryKey = primaryKeys[t] for (int i = 1; i <= 10; i++) { - sql """INSERT INTO ${pgDB}.${pgSchema}.${t} (id, tag, version) VALUES (${totalRows + i}, 'concurrent_ins', 1)""" + sql """INSERT INTO ${pgDB}.${pgSchema}.${t} (${primaryKey}, tag, version) VALUES (${totalRows + i}, 'concurrent_ins', 1)""" } - sql """UPDATE ${pgDB}.${pgSchema}.${t} SET version=99 WHERE id IN (1, 100, 500, 999)""" - sql """DELETE FROM ${pgDB}.${pgSchema}.${t} WHERE id IN (2, 200, 800)""" + sql """UPDATE ${pgDB}.${pgSchema}.${t} SET version=99 WHERE ${primaryKey} IN (1, 100, 500, 999)""" + sql """DELETE FROM ${pgDB}.${pgSchema}.${t} WHERE ${primaryKey} IN (2, 200, 800)""" } } @@ -112,17 +136,18 @@ suite("test_streaming_postgres_job_snapshot_with_concurrent_dml_multi_table", { boolean allOk = true for (def t : tables) { + def primaryKey = primaryKeys[t] def showTbl = sql """show tables from ${currentDb} like '${t}'""" if (showTbl.size() == 0) { allOk = false break } def cnt = sql """select count(1) from ${currentDb}.${t}""" - def upd1 = sql """select version from ${currentDb}.${t} where id=1""" - def upd999 = sql """select version from ${currentDb}.${t} where id=999""" - def del2 = sql """select count(1) from ${currentDb}.${t} where id=2""" - def del800 = sql """select count(1) from ${currentDb}.${t} where id=800""" - def ins = sql """select count(1) from ${currentDb}.${t} where id=${totalRows + 10}""" + def upd1 = sql """select version from ${currentDb}.${t} where ${primaryKey}=1""" + def upd999 = sql """select version from ${currentDb}.${t} where ${primaryKey}=999""" + def del2 = sql """select count(1) from ${currentDb}.${t} where ${primaryKey}=2""" + def del800 = sql """select count(1) from ${currentDb}.${t} where ${primaryKey}=800""" + def ins = sql """select count(1) from ${currentDb}.${t} where ${primaryKey}=${totalRows + 10}""" def v1 = upd1.size() == 0 ? null : upd1.get(0).get(0) def v999 = upd999.size() == 0 ? null : upd999.get(0).get(0) log.info("table=${t} cnt=${cnt} v1=${v1} v999=${v999} del2=${del2} del800=${del800} ins=${ins}") @@ -150,10 +175,10 @@ suite("test_streaming_postgres_job_snapshot_with_concurrent_dml_multi_table", qt_select_count_t1 """select count(1) from ${currentDb}.${table1}""" qt_select_count_t2 """select count(1) from ${currentDb}.${table2}""" - qt_select_updates_t1 """select id, version from ${currentDb}.${table1} where id in (1, 100, 500, 999) order by id""" - qt_select_updates_t2 """select id, version from ${currentDb}.${table2} where id in (1, 100, 500, 999) order by id""" - qt_select_inserts_t1 """select id, tag, version from ${currentDb}.${table1} where id > ${totalRows} order by id""" - qt_select_inserts_t2 """select id, tag, version from ${currentDb}.${table2} where id > ${totalRows} order by id""" + qt_select_updates_t1 """select pk, version from ${currentDb}.${table1} where pk in (1, 100, 500, 999) order by pk""" + qt_select_updates_t2 """select other_id, version from ${currentDb}.${table2} where other_id in (1, 100, 500, 999) order by other_id""" + qt_select_inserts_t1 """select pk, tag, version from ${currentDb}.${table1} where pk > ${totalRows} order by pk""" + qt_select_inserts_t2 """select other_id, tag, version from ${currentDb}.${table2} where other_id > ${totalRows} order by other_id""" sql """DROP JOB IF EXISTS where jobname = '${jobName}'""" diff --git a/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_cdc_stream_tvf_mysql.groovy b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_cdc_stream_tvf_mysql.groovy index ba6f0eacac32c4..744ab7539acb2c 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_cdc_stream_tvf_mysql.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_cdc_stream_tvf_mysql.groovy @@ -146,17 +146,20 @@ suite("test_cdc_stream_tvf_mysql", "p0,external,mysql,external_docker,external_d sql """CREATE TABLE ${mysqlDb}.${table1} ( `name` varchar(200) NOT NULL, `age` int DEFAULT NULL, + `tinyint_1` tinyint(1) DEFAULT NULL, + `year_zero` year DEFAULT NULL, PRIMARY KEY (`name`) ) ENGINE=InnoDB""" - sql """INSERT INTO ${mysqlDb}.${table1} (name, age) VALUES ('A1', 1);""" - sql """INSERT INTO ${mysqlDb}.${table1} (name, age) VALUES ('B1', 2);""" + sql """INSERT INTO ${mysqlDb}.${table1} VALUES ('A1', 1, 0, 0);""" + sql """INSERT INTO ${mysqlDb}.${table1} VALUES ('B1', 2, 1, 2023);""" def result = sql_return_maparray "show master status" def file = result[0]["File"] def position = result[0]["Position"] offset = """{"file":"${file}","pos":"${position}"}""" - sql """INSERT INTO ${mysqlDb}.${table1} (name, age) VALUES ('C1', 3);""" - sql """INSERT INTO ${mysqlDb}.${table1} (name, age) VALUES ('D1', 4);""" + // Values 4 and YEAR 0 distinguish numeric JDBC semantics from boolean/date semantics. + sql """INSERT INTO ${mysqlDb}.${table1} VALUES ('C1', 3, 4, 0);""" + sql """INSERT INTO ${mysqlDb}.${table1} VALUES ('D1', 4, -1, 2024);""" // capture offset before UPDATE/DELETE events def result2 = sql_return_maparray "show master status" diff --git a/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_mysql_jdbc_type_defaults.groovy b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_mysql_jdbc_type_defaults.groovy new file mode 100644 index 00000000000000..b246d653627882 --- /dev/null +++ b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_mysql_jdbc_type_defaults.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +/** + * Verify that a MySQL cdc_stream job uses numeric TINYINT(1) and YEAR semantics + * consistently while fetching snapshot splits and scanning snapshot/binlog rows. + */ +suite("test_streaming_job_cdc_stream_mysql_jdbc_type_defaults", + "p0,external,mysql,external_docker,external_docker_mysql,nondatalake") { + def jobName = "test_cdc_stream_mysql_jdbc_defaults_job" + def currentDb = (sql "select database()")[0][0] + + sql """DROP JOB IF EXISTS where jobname = '${jobName}'""" + sql """DROP TABLE IF EXISTS ${currentDb}.test_cdc_stream_mysql_jdbc_defaults_tbl FORCE""" + + sql """ + CREATE TABLE ${currentDb}.test_cdc_stream_mysql_jdbc_defaults_tbl ( + `name` varchar(32) NULL, + `tinyint_value` tinyint NULL, + `year_value` smallint NULL + ) ENGINE=OLAP + DUPLICATE KEY(`name`) + DISTRIBUTED BY HASH(`name`) BUCKETS AUTO + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + + String enabled = context.config.otherConfigs.get("enableJdbcTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String mysqlPort = context.config.otherConfigs.get("mysql_57_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3Endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driverUrl = "https://${bucket}.${s3Endpoint}/regression/jdbc_driver/mysql-connector-j-8.4.0.jar" + + connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysqlPort}") { + sql """CREATE DATABASE IF NOT EXISTS test_cdc_db""" + sql """DROP TABLE IF EXISTS test_cdc_db.test_cdc_stream_mysql_jdbc_defaults_src""" + sql """ + CREATE TABLE test_cdc_db.test_cdc_stream_mysql_jdbc_defaults_src ( + `name` varchar(32) NOT NULL, + `tinyint_value` tinyint(1) NOT NULL, + `year_value` year NOT NULL + ) ENGINE=InnoDB + """ + sql """INSERT INTO test_cdc_db.test_cdc_stream_mysql_jdbc_defaults_src VALUES ('A1', -1, 0)""" + sql """INSERT INTO test_cdc_db.test_cdc_stream_mysql_jdbc_defaults_src VALUES ('B1', 0, 2024)""" + sql """INSERT INTO test_cdc_db.test_cdc_stream_mysql_jdbc_defaults_src VALUES ('C1', 2, 0)""" + sql """INSERT INTO test_cdc_db.test_cdc_stream_mysql_jdbc_defaults_src VALUES ('D1', 4, 2025)""" + } + + sql """ + CREATE JOB ${jobName} + ON STREAMING DO INSERT INTO ${currentDb}.test_cdc_stream_mysql_jdbc_defaults_tbl + (name, tinyint_value, year_value) + SELECT name, tinyint_value, year_value FROM cdc_stream( + "type" = "mysql", + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysqlPort}", + "driver_url" = "${driverUrl}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "user" = "root", + "password" = "123456", + "database" = "test_cdc_db", + "table" = "test_cdc_stream_mysql_jdbc_defaults_src", + "offset" = "initial", + "snapshot_split_key" = "tinyint_value", + "snapshot_split_size" = "1" + ) + """ + + try { + Awaitility.await().atMost(300, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql """SELECT count(1) FROM ${currentDb}.test_cdc_stream_mysql_jdbc_defaults_tbl""" + log.info("snapshot rows: " + rows) + (rows.get(0).get(0) as int) == 4 + }) + } catch (Exception ex) { + log.info("job: " + (sql """select * from jobs("type"="insert") where Name='${jobName}'""")) + log.info("tasks: " + (sql """select * from tasks("type"="insert") where JobName='${jobName}'""")) + throw ex + } + + order_qt_snapshot_data """SELECT * FROM ${currentDb}.test_cdc_stream_mysql_jdbc_defaults_tbl""" + + connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysqlPort}") { + sql """INSERT INTO test_cdc_db.test_cdc_stream_mysql_jdbc_defaults_src VALUES ('E1', 127, 0)""" + sql """INSERT INTO test_cdc_db.test_cdc_stream_mysql_jdbc_defaults_src VALUES ('F1', -128, 2026)""" + } + + try { + Awaitility.await().atMost(120, SECONDS).pollInterval(2, SECONDS).until({ + def rows = sql """ + SELECT count(1) FROM ${currentDb}.test_cdc_stream_mysql_jdbc_defaults_tbl + WHERE name IN ('E1', 'F1') + """ + log.info("incremental rows: " + rows) + (rows.get(0).get(0) as int) == 2 + }) + } catch (Exception ex) { + log.info("job: " + (sql """select * from jobs("type"="insert") where Name='${jobName}'""")) + log.info("tasks: " + (sql """select * from tasks("type"="insert") where JobName='${jobName}'""")) + throw ex + } + + order_qt_final_data """SELECT * FROM ${currentDb}.test_cdc_stream_mysql_jdbc_defaults_tbl""" + sql """DROP JOB IF EXISTS where jobname = '${jobName}'""" + } +} diff --git a/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_latest_alter_cred.groovy b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_latest_alter_cred.groovy index f1cd5e13f91c63..87171eb369fdb1 100644 --- a/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_latest_alter_cred.groovy +++ b/regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_latest_alter_cred.groovy @@ -104,11 +104,14 @@ suite("test_streaming_job_cdc_stream_postgres_latest_alter_cred", ) """ - // wait for job to reach RUNNING state (at least one task scheduled) + // wait for the first task to commit the resolved latest offset try { - Awaitility.await().atMost(60, SECONDS).pollInterval(1, SECONDS).until({ - def status = sql """select status from jobs("type"="insert") where Name='${jobName}'""" - status.size() == 1 && status.get(0).get(0) == "RUNNING" + Awaitility.await().atMost(300, SECONDS).pollInterval(1, SECONDS).until({ + def jobInfo = sql """select Status, SucceedTaskCount from jobs("type"="insert") + where Name='${jobName}' and ExecuteType='STREAMING'""" + log.info("job readiness for latest offset: " + jobInfo) + jobInfo.size() == 1 && jobInfo.get(0).get(0) == "RUNNING" + && (jobInfo.get(0).get(1) as int) >= 1 }) } catch (Exception ex) { log.info("job: " + (sql """select * from jobs("type"="insert") where Name='${jobName}'""")) diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_first_error_msg.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_first_error_msg.groovy new file mode 100644 index 00000000000000..a41c84d2ce04a0 --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_first_error_msg.groovy @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.RoutineLoadTestUtils +import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.admin.NewTopic +import org.apache.kafka.clients.producer.ProducerConfig +import org.junit.Assert + +suite("test_routine_load_first_error_msg", "p0") { + if (!RoutineLoadTestUtils.isKafkaTestEnabled(context)) { + return + } + + def kafkaBroker = RoutineLoadTestUtils.getKafkaBroker(context) + def kafkaTopic = "test_routine_load_first_error_msg_${System.currentTimeMillis()}" + def jobName = "test_routine_load_first_error_msg" + + def adminProps = new Properties() + adminProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBroker) + def adminClient = AdminClient.create(adminProps) + try { + adminClient.createTopics([new NewTopic(kafkaTopic, 1, (short) 1)]).all().get() + } finally { + adminClient.close() + } + + try { + sql "DROP TABLE IF EXISTS test_routine_load_first_error_msg" + sql """ + CREATE TABLE test_routine_load_first_error_msg ( + id INT, + name STRING + ) + PARTITION BY RANGE(id) ( + PARTITION p0 VALUES LESS THAN ("10") + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + + sql """ + CREATE ROUTINE LOAD ${jobName} ON test_routine_load_first_error_msg + COLUMNS TERMINATED BY "|" + PROPERTIES ( + "max_error_number" = "10", + "max_filter_ratio" = "1.0", + "max_batch_interval" = "5" + ) + FROM KAFKA ( + "kafka_broker_list" = "${kafkaBroker}", + "kafka_topic" = "${kafkaTopic}", + "property.kafka_default_offsets" = "OFFSET_END" + ) + """ + + def count = 0 + while (true) { + def showResult = sql "SHOW ROUTINE LOAD FOR ${jobName}" + if (showResult[0][8].toString() == "RUNNING") { + break + } + if (count++ > 60) { + Assert.fail("Routine Load job did not enter RUNNING state") + } + sleep(1000) + } + + def producer = RoutineLoadTestUtils.createKafkaProducer(kafkaBroker) + try { + RoutineLoadTestUtils.sendTestDataToKafka( + producer, [kafkaTopic], ["100|bad_row", "1|valid_row"]) + producer.flush() + } finally { + producer.close() + } + + count = 0 + while (true) { + def jobInfo = sql """ + SELECT ERROR_LOG_URLS, FIRST_ERROR_MSG + FROM information_schema.routine_load_jobs + WHERE JOB_NAME = '${jobName}' + """ + def showResult = sql "SHOW ROUTINE LOAD FOR ${jobName}" + def loadedRows = sql "SELECT COUNT(*) FROM test_routine_load_first_error_msg" + def informationSchemaErrorLogUrls = jobInfo[0][0].toString() + def firstErrorMsg = jobInfo[0][1] + def showErrorLogUrls = showResult[0][18].toString() + def showFirstErrorMsg = showResult[0][23] + if (loadedRows[0][0] == 1 && firstErrorMsg != null + && firstErrorMsg.toString().contains("bad_row") + && showFirstErrorMsg != null + && showFirstErrorMsg.toString().contains("bad_row")) { + assertEquals(informationSchemaErrorLogUrls, showErrorLogUrls) + assertFalse(informationSchemaErrorLogUrls.contains("first_error_msg:")) + assertFalse(showErrorLogUrls.contains("first_error_msg:")) + assertEquals(firstErrorMsg.toString(), showFirstErrorMsg.toString()) + break + } + if (count++ > 60) { + Assert.fail("First error message was not reported by routine_load_jobs and SHOW ROUTINE LOAD: " + + "loadedRows=${loadedRows[0][0]}, firstErrorMsg=${firstErrorMsg}, " + + "showFirstErrorMsg=${showFirstErrorMsg}, " + + "informationSchemaErrorLogUrlsEmpty=${informationSchemaErrorLogUrls.isEmpty()}, " + + "showErrorLogUrlsEmpty=${showErrorLogUrls.isEmpty()}") + } + sleep(1000) + } + } finally { + try_sql "STOP ROUTINE LOAD FOR ${jobName}" + } +} diff --git a/regression-test/suites/meta_action_p0/test_dump_image.groovy b/regression-test/suites/meta_action_p0/test_dump_image.groovy index 4500503ae4267b..6102dfff2352be 100644 --- a/regression-test/suites/meta_action_p0/test_dump_image.groovy +++ b/regression-test/suites/meta_action_p0/test_dump_image.groovy @@ -17,6 +17,7 @@ suite("test_dump_image", "nonConcurrent") { httpTest { + basicAuthorization "${context.config.jdbcUser}", "${context.config.jdbcPassword}" endpoint context.config.feHttpAddress uri "/dump" op "get" diff --git a/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.groovy b/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.groovy index 33fb4f62db3a2f..d3400db766f4cb 100644 --- a/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.groovy +++ b/regression-test/suites/nereids_rules_p0/agg_strategy/distinct_agg_rewriter.groovy @@ -38,17 +38,12 @@ suite("distinct_agg_rewriter") { sql "set agg_phase=4" sql "select group_concat(distinct dst_key1 order by dst_key2) from t_gbykey_10_dstkey_10_1000_id group by gby_key" - test { - sql """select number % 2, count(distinct cast(number as varchar), cast(number as varchar)), - group_concat(distinct cast(number as varchar) order by number + 1) from numbers('number'='10') group by number % 2""" - exception "Unsupported query" - } - test { - sql """select count(distinct cast(number as varchar), cast(number as varchar)), - group_concat(distinct cast(number as varchar) order by number + 1) from numbers('number'='10')""" - exception "Unsupported query" - } + sql """select number % 2, count(distinct cast(number as varchar), cast(number as varchar)), + group_concat(distinct cast(number as varchar) order by number + 1) from numbers('number'='10') group by number % 2""" + + sql """select count(distinct cast(number as varchar), cast(number as varchar)), + group_concat(distinct cast(number as varchar) order by number + 1) from numbers('number'='10')""" // expect to have 3 HashAgg instead of 4 HashAgg sql "set agg_phase=3" diff --git a/regression-test/suites/nereids_rules_p0/expression/test_simplify_comparison_predicate.groovy b/regression-test/suites/nereids_rules_p0/expression/test_simplify_comparison_predicate.groovy index cdf5af5d481684..0495a4485b149f 100644 --- a/regression-test/suites/nereids_rules_p0/expression/test_simplify_comparison_predicate.groovy +++ b/regression-test/suites/nereids_rules_p0/expression/test_simplify_comparison_predicate.groovy @@ -54,10 +54,10 @@ suite('test_simplify_comparison_predicate', 'nonConcurrent') { checkExplain expression, resExpression } else { if (checkNullColumn) { - checkExplain expression.replace(column, "c_${type}_null"), resExpression.replace(column, "c_${type}_null") + checkExplain expression.replace(column, "test_simplify_comparison_predicate_tbl.c_${type}_null"), resExpression.replace(column, "test_simplify_comparison_predicate_tbl.c_${type}_null") } if (checkNotNullColumn) { - checkExplain expression.replace(column, "c_${type}"), resExpression.replace(column, "c_${type}") + checkExplain expression.replace(column, "test_simplify_comparison_predicate_tbl.c_${type}"), resExpression.replace(column, "test_simplify_comparison_predicate_tbl.c_${type}") } } } @@ -118,44 +118,44 @@ suite('test_simplify_comparison_predicate', 'nonConcurrent') { testSimplify true, true, '{int_like_column} >= 1.01', '({int_like_column} >= 2)' testSimplify true, true, '{int_like_column} <= 1.01', '({int_like_column} <= 1)' testSimplify true, true, '{int_like_column} < 1.01', '({int_like_column} < 2)' - testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) = CAST(1.00 as DECIMAL(10, 5))', '(c_decimal_3_0_null = 1)' - testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) = CAST(1.1 as DECIMAL(10, 5))', 'AND[c_decimal_3_0_null IS NULL,NULL]' - testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) > CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_3_0_null > 1)' - testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) >= CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_3_0_null >= 2)' - testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) < CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_3_0_null < 2)' - testSimplify false, false, 'CAST(c_decimal_3_0_null as DECIMAL(10, 5)) <= CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_3_0_null <= 1)' - testSimplify false, false, 'c_decimal_5_2_null = CAST(1.0 as DECIMAL(10, 5))', '(c_decimal_5_2_null = 1.00)' - testSimplify false, false, 'c_decimal_5_2_null = CAST(1.1 as DECIMAL(10, 5))', '(c_decimal_5_2_null = 1.10)' - testSimplify false, false, 'c_decimal_5_2_null = CAST(1.12 as DECIMAL(10, 5))', '(c_decimal_5_2_null = 1.12)' - testSimplify false, false, 'c_decimal_5_2_null = CAST(1.123 as DECIMAL(10, 5))', 'AND[c_decimal_5_2_null IS NULL,NULL]' - testSimplify false, false, 'c_decimal_5_2 = CAST(1.123 as DECIMAL(10, 5))', 'FALSE' - testSimplify false, false, 'c_decimal_5_2_null > CAST(1.123 as DECIMAL(10, 5))', '(c_decimal_5_2_null > 1.12)' - testSimplify false, false, 'c_decimal_5_2_null >= CAST(1.123 as DECIMAL(10, 5))', '(c_decimal_5_2_null >= 1.13)' - testSimplify false, false, 'c_decimal_5_2_null <= CAST(1.123 as DECIMAL(10, 5))', '(c_decimal_5_2_null <= 1.12)' - testSimplify false, false, 'c_decimal_5_2_null < CAST(1.123 as DECIMAL(10, 5))', '(c_decimal_5_2_null < 1.13)' - testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) = '2000-01-01'", "(c_datetime_0 = '2000-01-01 00:00:00')" - testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) = '2000-01-01 00:00:00.1'", 'FALSE' - testSimplify false, false, "CAST(c_datetime_0_null AS DATETIME(5)) = '2000-01-01 00:00:00.1'", 'AND[c_datetime_0_null IS NULL,NULL]' - testSimplify false, false, "CAST(c_datetime_0_null AS DATETIME(5)) <=> '2000-01-01 00:00:00.1'", 'FALSE' - testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) >= '2000-01-01 00:00:00.1'", "(c_datetime_0 >= '2000-01-01 00:00:01')" - testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) > '2000-01-01 00:00:00.1'", "(c_datetime_0 > '2000-01-01 00:00:00')" - testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) <= '2000-01-01 00:00:00.1'", "(c_datetime_0 <= '2000-01-01 00:00:00')" - testSimplify false, false, "CAST(c_datetime_0 AS DATETIME(5)) < '2000-01-01 00:00:00.1'", "(c_datetime_0 < '2000-01-01 00:00:01')" - testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) = '2000-01-01'", "(c_datetime_3 = '2000-01-01 00:00:00.000')" - testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) = '2000-01-01 00:00:00.1234'", 'FALSE' - testSimplify false, false, "CAST(c_datetime_3_null AS DATETIME(5)) = '2000-01-01 00:00:00.1234'", 'AND[c_datetime_3_null IS NULL,NULL]' - testSimplify false, false, "CAST(c_datetime_3_null AS DATETIME(5)) <=> '2000-01-01 00:00:00.1234'", 'FALSE' - testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) >= '2000-01-01 00:00:00.1234'", "(c_datetime_3 >= '2000-01-01 00:00:00.124')" - testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) > '2000-01-01 00:00:00.1234'", "(c_datetime_3 > '2000-01-01 00:00:00.123')" - testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) <= '2000-01-01 00:00:00.1234'", "(c_datetime_3 <= '2000-01-01 00:00:00.123')" - testSimplify false, false, "CAST(c_datetime_3 AS DATETIME(5)) < '2000-01-01 00:00:00.1234'", "(c_datetime_3 < '2000-01-01 00:00:00.124')" + testSimplify false, false, 'CAST(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null as DECIMAL(10, 5)) = CAST(1.00 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null = 1)' + testSimplify false, false, 'CAST(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null as DECIMAL(10, 5)) = CAST(1.1 as DECIMAL(10, 5))', 'AND[test_simplify_comparison_predicate_tbl.c_decimal_3_0_null IS NULL,NULL]' + testSimplify false, false, 'CAST(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null as DECIMAL(10, 5)) > CAST(1.1 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null > 1)' + testSimplify false, false, 'CAST(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null as DECIMAL(10, 5)) >= CAST(1.1 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null >= 2)' + testSimplify false, false, 'CAST(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null as DECIMAL(10, 5)) < CAST(1.1 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null < 2)' + testSimplify false, false, 'CAST(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null as DECIMAL(10, 5)) <= CAST(1.1 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_3_0_null <= 1)' + testSimplify false, false, 'test_simplify_comparison_predicate_tbl.c_decimal_5_2_null = CAST(1.0 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_5_2_null = 1.00)' + testSimplify false, false, 'test_simplify_comparison_predicate_tbl.c_decimal_5_2_null = CAST(1.1 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_5_2_null = 1.10)' + testSimplify false, false, 'test_simplify_comparison_predicate_tbl.c_decimal_5_2_null = CAST(1.12 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_5_2_null = 1.12)' + testSimplify false, false, 'test_simplify_comparison_predicate_tbl.c_decimal_5_2_null = CAST(1.123 as DECIMAL(10, 5))', 'AND[test_simplify_comparison_predicate_tbl.c_decimal_5_2_null IS NULL,NULL]' + testSimplify false, false, 'test_simplify_comparison_predicate_tbl.c_decimal_5_2 = CAST(1.123 as DECIMAL(10, 5))', 'FALSE' + testSimplify false, false, 'test_simplify_comparison_predicate_tbl.c_decimal_5_2_null > CAST(1.123 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_5_2_null > 1.12)' + testSimplify false, false, 'test_simplify_comparison_predicate_tbl.c_decimal_5_2_null >= CAST(1.123 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_5_2_null >= 1.13)' + testSimplify false, false, 'test_simplify_comparison_predicate_tbl.c_decimal_5_2_null <= CAST(1.123 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_5_2_null <= 1.12)' + testSimplify false, false, 'test_simplify_comparison_predicate_tbl.c_decimal_5_2_null < CAST(1.123 as DECIMAL(10, 5))', '(test_simplify_comparison_predicate_tbl.c_decimal_5_2_null < 1.13)' + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_0 AS DATETIME(5)) = '2000-01-01'", "(test_simplify_comparison_predicate_tbl.c_datetime_0 = '2000-01-01 00:00:00')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_0 AS DATETIME(5)) = '2000-01-01 00:00:00.1'", 'FALSE' + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_0_null AS DATETIME(5)) = '2000-01-01 00:00:00.1'", 'AND[test_simplify_comparison_predicate_tbl.c_datetime_0_null IS NULL,NULL]' + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_0_null AS DATETIME(5)) <=> '2000-01-01 00:00:00.1'", 'FALSE' + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_0 AS DATETIME(5)) >= '2000-01-01 00:00:00.1'", "(test_simplify_comparison_predicate_tbl.c_datetime_0 >= '2000-01-01 00:00:01')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_0 AS DATETIME(5)) > '2000-01-01 00:00:00.1'", "(test_simplify_comparison_predicate_tbl.c_datetime_0 > '2000-01-01 00:00:00')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_0 AS DATETIME(5)) <= '2000-01-01 00:00:00.1'", "(test_simplify_comparison_predicate_tbl.c_datetime_0 <= '2000-01-01 00:00:00')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_0 AS DATETIME(5)) < '2000-01-01 00:00:00.1'", "(test_simplify_comparison_predicate_tbl.c_datetime_0 < '2000-01-01 00:00:01')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_3 AS DATETIME(5)) = '2000-01-01'", "(test_simplify_comparison_predicate_tbl.c_datetime_3 = '2000-01-01 00:00:00.000')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_3 AS DATETIME(5)) = '2000-01-01 00:00:00.1234'", 'FALSE' + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_3_null AS DATETIME(5)) = '2000-01-01 00:00:00.1234'", 'AND[test_simplify_comparison_predicate_tbl.c_datetime_3_null IS NULL,NULL]' + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_3_null AS DATETIME(5)) <=> '2000-01-01 00:00:00.1234'", 'FALSE' + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_3 AS DATETIME(5)) >= '2000-01-01 00:00:00.1234'", "(test_simplify_comparison_predicate_tbl.c_datetime_3 >= '2000-01-01 00:00:00.124')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_3 AS DATETIME(5)) > '2000-01-01 00:00:00.1234'", "(test_simplify_comparison_predicate_tbl.c_datetime_3 > '2000-01-01 00:00:00.123')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_3 AS DATETIME(5)) <= '2000-01-01 00:00:00.1234'", "(test_simplify_comparison_predicate_tbl.c_datetime_3 <= '2000-01-01 00:00:00.123')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_datetime_3 AS DATETIME(5)) < '2000-01-01 00:00:00.1234'", "(test_simplify_comparison_predicate_tbl.c_datetime_3 < '2000-01-01 00:00:00.124')" testSimplify false, false, "c_date = '2000-01-01 00:00:01'", 'FALSE' - testSimplify false, false, "CAST(c_date_null AS DATETIME(5)) = '2000-01-01 00:00:01'", 'AND[c_date_null IS NULL,NULL]' - testSimplify false, false, "CAST(c_date_null AS DATETIME(5)) <=> '2000-01-01 00:00:01'", 'FALSE' - testSimplify false, false, "CAST(c_date AS DATETIME(5)) > '2000-01-01 00:00:01'", "(c_date > '2000-01-01')" - testSimplify false, false, "CAST(c_date AS DATETIME(5)) >= '2000-01-01 00:00:01'", "(c_date >= '2000-01-02')" - testSimplify false, false, "CAST(c_date AS DATETIME(5)) <= '2000-01-01 00:00:01'", "(c_date <= '2000-01-01')" - testSimplify false, false, "CAST(c_date AS DATETIME(5)) < '2000-01-01 00:00:01'", "(c_date < '2000-01-02')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_date_null AS DATETIME(5)) = '2000-01-01 00:00:01'", 'AND[test_simplify_comparison_predicate_tbl.c_date_null IS NULL,NULL]' + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_date_null AS DATETIME(5)) <=> '2000-01-01 00:00:01'", 'FALSE' + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_date AS DATETIME(5)) > '2000-01-01 00:00:01'", "(test_simplify_comparison_predicate_tbl.c_date > '2000-01-01')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_date AS DATETIME(5)) >= '2000-01-01 00:00:01'", "(test_simplify_comparison_predicate_tbl.c_date >= '2000-01-02')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_date AS DATETIME(5)) <= '2000-01-01 00:00:01'", "(test_simplify_comparison_predicate_tbl.c_date <= '2000-01-01')" + testSimplify false, false, "CAST(test_simplify_comparison_predicate_tbl.c_date AS DATETIME(5)) < '2000-01-01 00:00:01'", "(test_simplify_comparison_predicate_tbl.c_date < '2000-01-02')" sql "DROP TABLE IF EXISTS ${tbl} FORCE" } diff --git a/regression-test/suites/nereids_rules_p0/infer_predicate/infer_intersect_except.groovy b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_intersect_except.groovy index 9aa5ce723e5d2d..cd18ade2b04517 100644 --- a/regression-test/suites/nereids_rules_p0/infer_predicate/infer_intersect_except.groovy +++ b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_intersect_except.groovy @@ -84,22 +84,22 @@ suite("infer_intersect_except") { insert into infer_intersect_except3 values(1,'d2',2,2),(-2,'d2',2,2),(0,'d2',3,5); """ qt_except """ - explain shape plan + explain shape plan select a,b from infer_intersect_except1 where a>0 except select a,b from infer_intersect_except2 where b>'ab' except select a,b from infer_intersect_except2 where a<10; """ qt_except_to_empty """ - explain shape plan + explain shape plan select a,b from infer_intersect_except1 where a>0 except select a,b from infer_intersect_except2 where b>'ab' except select a,b from infer_intersect_except3 where a<0; """ qt_except_not_infer_1_greater_than_0 """ - explain shape plan + explain shape plan select a,b from infer_intersect_except1 where a>0 except select 1,'abc' from infer_intersect_except2 where b>'ab' except select a,b from infer_intersect_except2 where a<0; """ qt_except_number_and_string """ - explain shape plan + explain shape plan select a,2 from infer_intersect_except1 where a>0 except select 1,'abc' from infer_intersect_except2 where b>'ab' except select a,b from infer_intersect_except3 where a<0; """ @@ -127,7 +127,7 @@ suite("infer_intersect_except") { select a, b from infer_intersect_except3 where a < 10; """ notContains("a < 10") - contains("cast(b as TEXT) = 'abc'") + contains("(cast(infer_intersect_except3.b as TEXT) = 'abc'") contains("infer_intersect_except3.a = 1") // PhysicalResultSink // --PhysicalExcept diff --git a/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy b/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy index 4ee14eba4817f1..356bf9e5fd401c 100644 --- a/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy +++ b/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy @@ -18,6 +18,17 @@ import java.util.stream.Collectors // under the License. suite("shuffle_left_join") { + // The point of this suite is the left-to-right bucket shuffle: with the nereids distribute + // planner on, the aggregated left side is shuffled onto the right table's storage buckets, so + // the join becomes a bucket shuffle join that saves one exchange versus a plain partitioned + // shuffle. The planner chooses between the two candidates by cost, which depends on the scan + // row count and on the bucket-shuffle downgrade gate. Both are pinned below so the asserted + // plan is stable: + // - `analyze ... with sync` fixes the row count (otherwise it is reported asynchronously + // after the insert, and the plan flips depending on whether the report has landed yet); + // - `bucket_shuffle_downgrade_ratio=0` disables the downgrade that turns bucket shuffle back + // into a partitioned shuffle when the bucket count is small relative to the instance count, + // which otherwise makes the plan depend on the number of backends. multi_sql """ drop table if exists test_shuffle_left; @@ -35,11 +46,14 @@ suite("shuffle_left_join") { sync; + analyze table test_shuffle_left with sync; + set enable_nereids_distribute_planner=false; set enable_pipeline_x_engine=true; set disable_join_reorder=true; set enable_local_shuffle=false; set force_to_local_shuffle=false; + set bucket_shuffle_downgrade_ratio=0; """ def extractFragment = { String sqlStr, String containsString, Closure checkExchangeNum -> @@ -95,8 +109,8 @@ suite("shuffle_left_join") { .collect(Collectors.joining("\n")) logger.info("Variables:\n${variableString}") - extractFragment(sqlStr, "INNER JOIN(PARTITIONED)") { exchangeNum -> - assertTrue(exchangeNum == 2) + extractFragment(sqlStr, "INNER JOIN(BUCKET_SHUFFLE)") { exchangeNum -> + assertTrue(exchangeNum == 1) } explain { diff --git a/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy b/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy index 4b71d29ca53374..e2b0010e7492ef 100644 --- a/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy +++ b/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy @@ -480,7 +480,7 @@ suite("agg_sync_mv") { streamLoad { table "agg_mv_test" - db "regression_test_nereids_syntax_p1_mv" + db "regression_test_nereids_syntax_p1_mv_aggregate" set 'column_separator', ';' set 'columns', ''' id, kbool, ktint, ksint, kint, kbint, klint, kfloat, kdbl, kdcmls1, kdcmls2, kdcmls3, @@ -493,7 +493,7 @@ suite("agg_sync_mv") { km_tint_bool, km_int_int, km_tint_sint, km_tint_int, km_tint_bint, km_tint_lint, km_tint_float, km_tint_dbl, km_tint_dcml, km_tint_chr, km_tint_vchr, km_tint_str, km_tint_date, km_tint_dtm, kjson, kstruct ''' - file "../agg_mv_test.dat" + file "agg_mv_test.dat" } diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy index ad6bc3db160d05..f1d9ad0e077afc 100644 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy @@ -116,31 +116,45 @@ suite("test_auto_dynamic", "nonConcurrent") { part_result = sql " show partitions from auto_dynamic " assertEquals(part_result.size(), 1) - def skip_test = false - test { - sql " insert into auto_dynamic values ('2024-01-01'), ('2900-01-01'), ('1900-01-01'), ('3000-01-01'); " - check { result, exception, startTime, endTime -> - if (exception != null) { - // the partition of 1900-01-01 directly been recovered before the insert txn finished. let it success - part_result = sql " show partitions from auto_dynamic " - log.info("${part_result}".toString()) - assertTrue(exception.getMessage().contains("get partition p19000101000000 failed")) - skip_test = true + def oldCheckInterval = getFeConfig("dynamic_partition_check_interval_seconds") + try { + def insertException = null + test { + sql " insert into auto_dynamic values ('2024-01-01'), ('2900-01-01'), ('1900-01-01'), ('3000-01-01'); " + check { result, exception, startTime, endTime -> + insertException = exception } } - } - if (skip_test) { - return true - } - - sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '1') """ - sleep(10000) - part_result = sql " show partitions from auto_dynamic " - log.info("${part_result}".toString()) - assertTrue(part_result.size() == 3 || part_result.size() == 4, - "The partition size should be 3 or 4, but got ${part_result.size()}") + if (insertException != null) { + part_result = sql " show partitions from auto_dynamic " + def partitionNames = part_result.collect { it[1] } + def expectedSurvivingPartitions = ["p20240101000000", "p29000101000000", "p30000101000000"] + // A stale short-interval run can only delete the out-of-range p1900 partition before commit. + assertTrue(!partitionNames.contains("p19000101000000") + && partitionNames.containsAll(expectedSurvivingPartitions), + "Unexpected insert failure: ${insertException}, partitions: ${partitionNames}") + return true + } - qt_sql_dynamic_auto "select * from auto_dynamic order by k0;" + setFeConfig("dynamic_partition_check_interval_seconds", 1) + def partitionNames = [] + for (int retry = 0; retry < 1200; retry++) { + part_result = sql " show partitions from auto_dynamic " + partitionNames = part_result.collect { it[1] } + if (!partitionNames.contains("p19000101000000")) { + break + } + sleep(1000) + } + // Changing the interval does not wake a scheduler that is already sleeping. + assertTrue(!partitionNames.contains("p19000101000000"), + "Dynamic partition scheduler did not recycle p1900 within 1200 seconds, partitions: ${partitionNames}") + log.info("${part_result}".toString()) + assertTrue(part_result.size() == 3 || part_result.size() == 4, + "The partition size should be 3 or 4, but got ${part_result.size()}") - sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '600') """ -} \ No newline at end of file + qt_sql_dynamic_auto "select * from auto_dynamic order by k0;" + } finally { + setFeConfig("dynamic_partition_check_interval_seconds", oldCheckInterval) + } +} diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_new_recycle.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_new_recycle.groovy index b37437bdcbf3f2..83b71d2ca2c31d 100644 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_new_recycle.groovy +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_new_recycle.groovy @@ -111,8 +111,6 @@ suite("test_auto_new_recycle", "nonConcurrent") { ); """ - waitUntilSafeExecutionTime("NOT_CROSS_DAY_BOUNDARY", 20) - sql """ insert into auto_recycle select date_add('2020-01-01 00:00:00', interval number day) from numbers("number" = "100"); """ @@ -171,6 +169,7 @@ suite("test_auto_new_recycle", "nonConcurrent") { """ sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '600') """ sleep(8000) + waitUntilSafeExecutionTime("NOT_CROSS_DAY_BOUNDARY", 100) sql """ insert into auto_recycle select date_add(now(), interval number-5 day) from numbers("number" = "8"); """ @@ -182,4 +181,4 @@ suite("test_auto_new_recycle", "nonConcurrent") { assertEquals(res.size(), 6) sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '600') """ -} \ No newline at end of file +} diff --git a/regression-test/suites/query_p0/aggregate/agg_group_concat.groovy b/regression-test/suites/query_p0/aggregate/agg_group_concat.groovy index a05a9edb271629..ddafd4f214818d 100644 --- a/regression-test/suites/query_p0/aggregate/agg_group_concat.groovy +++ b/regression-test/suites/query_p0/aggregate/agg_group_concat.groovy @@ -105,6 +105,14 @@ suite("agg_group_concat") { from (select 1 as k union all select 2) t; """ + test { + sql """ + select group_concat(k order by sum(k)) as s + from (select 1 as k union all select 2) t; + """ + exception "aggregate function cannot contain aggregate parameters" + } + order_qt_group_concat_order_by_subquery """ select group_concat(kint order by (select 1), kint) as s from agg_group_concat_table; diff --git a/regression-test/suites/query_p0/cache/query_cache_incremental.groovy b/regression-test/suites/query_p0/cache/query_cache_incremental.groovy new file mode 100644 index 00000000000000..9ee2d876550e3c --- /dev/null +++ b/regression-test/suites/query_p0/cache/query_cache_incremental.groovy @@ -0,0 +1,321 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Correctness of the query cache incremental merge: when a partition keeps +// receiving hourly loads, a stale cache entry is reused by scanning only the +// delta rowsets since the cached version and merging them with the cached +// partial aggregation blocks. Every query below is checked against the same +// query with the cache disabled, so results stay verified even where +// incremental merge falls back to a full recompute (e.g. cloud mode). On +// local storage the suite additionally proves through the BE metrics that +// the incremental path really fires: the early stale queries must increase +// query_cache_stale_hit_total, and the two designed fallback phases (a +// delete predicate in the delta, a merge-on-write backfill that rewrites +// history) must increase query_cache_incremental_fallback_total. The final +// phase covers a long-dormant entry whose first reuse faces a large, +// many-rowset delta that goes through the parallel scanner builder. +suite("query_cache_incremental") { + def querySql = """ + SELECT + url, + SUM(cost) AS total_cost, + COUNT(*) AS cnt + FROM test_query_cache_incremental + WHERE dt >= '2026-01-01' + AND dt < '2026-01-15' + GROUP BY url + """ + + def normalize = { rows -> + return rows.collect { row -> row.collect { col -> String.valueOf(col) }.join("|") }.sort() + } + + // Compare the cached query result against the uncached one, twice: the + // first cached run may fill or incrementally merge the entry, the second + // one should serve it. + def checkConsistency = { String sqlText -> + sql "set enable_query_cache=false" + def expected = normalize(sql(sqlText)) + sql "set enable_query_cache=true" + assertEquals(expected, normalize(sql(sqlText))) + assertEquals(expected, normalize(sql(sqlText))) + } + + // Sum a BE counter across all backends via the webserver /metrics text. + def sumBeMetric = { String metricName -> + def backendIdToIp = [:] + def backendIdToHttpPort = [:] + getBackendIpHttpPort(backendIdToIp, backendIdToHttpPort) + long total = 0 + backendIdToIp.each { backendId, ip -> + def text = new URL("http://${ip}:${backendIdToHttpPort[backendId]}/metrics").getText() + for (def line : text.split("\n")) { + if (line.startsWith("doris_be_${metricName} ")) { + total += Long.parseLong(line.substring(line.lastIndexOf(' ') + 1).trim()) + } + } + } + return total + } + + // Besides result consistency, prove on local storage that the stale entry + // was really merged incrementally (Mode::INCREMENTAL), not recomputed: + // only the incremental path increases query_cache_stale_hit_total, and no + // concurrent suite touches it because the session switch defaults to off. + // Cloud mode always falls back by design, so it only checks consistency. + // Residual caveat (here and in checkIncrementalFallback): a + // memory-pressure prune evicting the entry inside the tiny fill-to-assert + // window surfaces as a plain MISS and would fail the assertion; accepted + // as rare, and a rerun re-establishes the counters from fresh deltas. + def checkStaleIncremental = { String sqlText -> + if (isCloudMode()) { + checkConsistency(sqlText) + return + } + def before = sumBeMetric("query_cache_stale_hit_total") + checkConsistency(sqlText) + def after = sumBeMetric("query_cache_stale_hit_total") + assertTrue(after > before, + "expected a stale incremental cache hit, but query_cache_stale_hit_total stayed at ${before}") + } + + // Prove that a designed fallback phase really took the fallback path. + def checkIncrementalFallback = { String sqlText -> + if (isCloudMode()) { + checkConsistency(sqlText) + return + } + def before = sumBeMetric("query_cache_incremental_fallback_total") + checkConsistency(sqlText) + def after = sumBeMetric("query_cache_incremental_fallback_total") + assertTrue(after > before, + "expected an incremental fallback, but query_cache_incremental_fallback_total stayed at ${before}") + } + + sql "set enable_nereids_distribute_planner=true" + sql "set enable_query_cache=true" + sql "set enable_query_cache_incremental=true" + sql "set parallel_pipeline_task_num=3" + sql "set enable_sql_cache=false" + + sql "DROP TABLE IF EXISTS test_query_cache_incremental" + sql """ + CREATE TABLE test_query_cache_incremental ( + dt DATE, + user_id INT, + url STRING, + cost BIGINT + ) + ENGINE=OLAP + DUPLICATE KEY(dt, user_id) + PARTITION BY RANGE(dt) + ( + PARTITION p20260101 VALUES LESS THAN ("2026-01-05"), + PARTITION p20260105 VALUES LESS THAN ("2026-01-10"), + PARTITION p20260110 VALUES LESS THAN ("2026-01-15") + ) + DISTRIBUTED BY HASH(user_id) BUCKETS 3 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + + sql """ + INSERT INTO test_query_cache_incremental VALUES + ('2026-01-01',1,'/a',10), + ('2026-01-01',2,'/b',20), + ('2026-01-02',3,'/c',30), + ('2026-01-06',1,'/a',15), + ('2026-01-07',3,'/c',35), + ('2026-01-11',1,'/a',50), + ('2026-01-12',3,'/c',70) + """ + + // The query cache participates in this plan. + explain { + sql(querySql) + contains("DIGEST") + } + + // Fill the cache, then serve from it. + checkConsistency(querySql) + order_qt_dup_initial "${querySql}" + + // Simulate the hourly load pattern: only the latest partition receives new + // data, so the entries of the older partitions stay valid while the hot + // partition entry is stale and gets incrementally merged. Crossing + // query_cache_max_incremental_merge_count (BE config, default 8) also + // exercises the forced full recompute that compacts the entry. + for (int i = 1; i <= 10; i++) { + sql """ + INSERT INTO test_query_cache_incremental VALUES + ('2026-01-13',${i},'/a',${i}), + ('2026-01-13',${100 + i},'/inc',${10 * i}) + """ + if (i == 1) { + // The hot partition holds just two data rowsets here, far from + // both the merge-count threshold and any compaction, so on local + // storage this stale query must take the incremental path. + checkStaleIncremental(querySql) + } else { + // Later rounds may legitimately recompute in full (merge-count + // threshold, background compaction), so assert correctness only. + checkConsistency(querySql) + } + } + order_qt_dup_after_appends "${querySql}" + + // A delete in the delta cannot be merged into the cached partial result; + // the query must fall back to a full recompute and stay correct. + sql "DELETE FROM test_query_cache_incremental PARTITION p20260110 WHERE user_id = 1" + checkIncrementalFallback(querySql) + + sql """ + INSERT INTO test_query_cache_incremental VALUES + ('2026-01-14',999,'/after-delete',1) + """ + checkConsistency(querySql) + order_qt_dup_final "${querySql}" + + // A UNIQUE merge-on-write table takes the incremental path as long as the + // loads only append new keys (the delete bitmap of the delta window stays + // empty); a load that rewrites a pre-existing key falls back to one full + // recompute and re-bases the entry, after which pure appends are + // incremental again. Results must stay correct in every phase. + sql "DROP TABLE IF EXISTS test_query_cache_incremental_mow" + sql """ + CREATE TABLE test_query_cache_incremental_mow ( + dt DATE, + user_id INT, + cost BIGINT + ) + ENGINE=OLAP + UNIQUE KEY(dt, user_id) + PARTITION BY RANGE(dt) + ( + PARTITION p20260101 VALUES LESS THAN ("2026-01-05"), + PARTITION p20260105 VALUES LESS THAN ("2026-01-10") + ) + DISTRIBUTED BY HASH(user_id) BUCKETS 3 + PROPERTIES + ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + def uniqueQuerySql = """ + SELECT dt, SUM(cost) AS total_cost + FROM test_query_cache_incremental_mow + GROUP BY dt + """ + sql """ + INSERT INTO test_query_cache_incremental_mow VALUES + ('2026-01-01',1,10), + ('2026-01-02',2,20), + ('2026-01-06',3,30) + """ + checkConsistency(uniqueQuerySql) + order_qt_mow_initial "${uniqueQuerySql}" + // The hourly-append pattern on a primary-key table: every load only adds + // brand-new keys, so the stale entry merges incrementally. The first + // round is asserted through the metric (two data rowsets, nothing can + // interfere); later rounds approach the compaction threshold, so they + // assert correctness only. + for (int i = 1; i <= 3; i++) { + sql "INSERT INTO test_query_cache_incremental_mow VALUES ('2026-01-06',${100 + i},${i})" + if (i == 1) { + checkStaleIncremental(uniqueQuerySql) + } else { + checkConsistency(uniqueQuerySql) + } + } + // A backfill rewrites history through the delete bitmap: user_id 1 gets a + // new cost, which must fall back to one full recompute that re-bases the + // entry. + sql "INSERT INTO test_query_cache_incremental_mow VALUES ('2026-01-01',1,100)" + checkIncrementalFallback(uniqueQuerySql) + // After the re-base, pure appends take the incremental path again (not + // asserted through the metric: by now enough rowsets piled up that a + // background compaction could legitimately force a full recompute). + sql "INSERT INTO test_query_cache_incremental_mow VALUES ('2026-01-06',200,7)" + checkConsistency(uniqueQuerySql) + order_qt_mow_final "${uniqueQuerySql}" + + // A cache entry can sit dormant while loads keep landing: the merge-count + // threshold bounds prior write-backs, not the versions accumulated while + // the entry is idle, so the delta of the first stale query after a long + // idle stretch can be arbitrarily large. Such a delta must flow through + // the parallel scanner builder like any full scan and still merge + // correctly with the cached blocks. Both settings below are needed for + // the split to actually happen at regression scale: the per-scanner row + // floor defaults to 2M rows, and BE clamps this variable up to 1024, so + // 1024 is the lowest floor reachable and the delta further down is sized + // several times past it. + sql "set enable_parallel_scan=true" + sql "set parallel_scan_min_rows_per_scanner=16" + sql "DROP TABLE IF EXISTS test_query_cache_incremental_dormant" + sql """ + CREATE TABLE test_query_cache_incremental_dormant ( + dt DATE, + user_id INT, + url STRING, + cost BIGINT + ) + ENGINE=OLAP + DUPLICATE KEY(dt, user_id) + PARTITION BY RANGE(dt) + ( + PARTITION p20260101 VALUES LESS THAN ("2026-01-15") + ) + DISTRIBUTED BY HASH(user_id) BUCKETS 1 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + def dormantQuerySql = """ + SELECT + url, + SUM(cost) AS total_cost, + COUNT(*) AS cnt + FROM test_query_cache_incremental_dormant + WHERE dt >= '2026-01-01' + AND dt < '2026-01-15' + GROUP BY url + """ + sql "INSERT INTO test_query_cache_incremental_dormant VALUES ('2026-01-01',1,'/a',10)" + // Fill the entry at the small initial version, then leave it dormant while + // 30 loads land on the single-tablet partition: a 30-rowset, 6000-row + // suffix. Against the 1024-row floor the builder cuts that into four or + // five scanners, so the delta really goes through the rowset/segment split + // rather than through one serial scanner. + checkConsistency(dormantQuerySql) + for (int i = 1; i <= 30; i++) { + def values = (1..200).collect { n -> + "('2026-01-0${(n % 7) + 2}',${i * 1000 + n},'/u${n % 5}',${n})" + }.join(",") + sql "INSERT INTO test_query_cache_incremental_dormant VALUES ${values}" + } + // The first query after the idle stretch must still take the incremental + // path on local storage and agree with the uncached result. The loads + // landed moments ago, so a boundary-crossing base compaction inside this + // window is as unlikely as in the first-round assertions above (an + // in-window cumulative compaction keeps the capture, and therefore the + // assertion, intact). + checkStaleIncremental(dormantQuerySql) +} diff --git a/regression-test/suites/query_p0/eager_agg/scalar_agg_pushdown.groovy b/regression-test/suites/query_p0/eager_agg/scalar_agg_pushdown.groovy new file mode 100644 index 00000000000000..64c502184feec6 --- /dev/null +++ b/regression-test/suites/query_p0/eager_agg/scalar_agg_pushdown.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("scalar_agg_pushdown") { + sql "set eager_aggregation_mode=1;" + sql "set eager_aggregation_on_join=true;" + sql "set disable_nereids_rules='SALT_JOIN';" + sql "set runtime_filter_mode=OFF;" + sql 'set ignore_shape_nodes="PhysicalProject, PhysicalDistribute";' + + // Test: scalar aggregation (no GROUP BY) must not be pushed below a join, + // because scalar agg on empty input returns 1 NULL row (SQL standard), + // which would be cross-joined with the other side to produce phantom rows. + // Regression test for: scalar agg pushdown producing phantom rows when + // constant propagation replaces GROUP BY key with a literal that doesn't + // exist in the table. + + sql """drop table if exists t1;""" + sql """create table t1 (a int, b int) distributed by hash(a) properties("replication_num" = "1");""" + sql """insert into t1 values (1,10),(2,20);""" + + sql """drop table if exists t2;""" + sql """create table t2 (c int) distributed by hash(c) properties("replication_num" = "1");""" + sql """insert into t2 values (1),(2);""" + + // WHERE t1.a = 999 matches no rows in t1. + // GROUP BY includes t1.a, t2.c (cross-table GROUP BY). + // Without the fix, eager aggregation pushes a scalar partial aggregate + // below the join, which on empty input returns 1 NULL row. + // That NULL row cross-joins with t2's 2 rows, producing 2 phantom rows. + // With the fix, pushdown is forbidden for scalar child aggregates. + qt_scalar_agg_no_pushdown """ + SELECT t1.a, SUM(t1.b), t2.c + FROM t1, t2 + WHERE t1.a = 999 + GROUP BY t1.a, t2.c + ORDER BY t2.c; + """ + + // Test: reviewer's example — constant group key k survives ConstantPropagation + // (retained when it is the only GROUP BY key, to keep group-by-with-literal + // semantics), then goes through a Project where createContextFromProject + // resolves k to empty input slots. sum(z) = sum(ta.b + tb.b) spans both + // inner-join children, so visitLogicalJoin falls back to genAggregate with + // groupKeys=[]. Plan after subquery unnesting (t1 self-joined as ta/tb, + // t2 as the cross side): + // + // Aggregate(group=[k], sum(z)) + // CrossJoin + // Project(1 AS k, ta.b + tb.b AS z) + // InnerJoin(ta.a = tb.a) + // Scan t1 ta + // Scan t1 tb + // Scan t2 + // + // WHERE ta.a = 999 empties the inner join. Without the guard at + // genAggregate(), a scalar aggregate is created above the inner join; + // its 0->1 row semantic produces a phantom row that cross-joins with + // t2's rows, yielding (1, NULL) instead of an empty result. + + // no scalar aggregate is allowed between the inner join and the cross join + qt_const_group_key_shape """ + explain shape plan + SELECT k, SUM(z) + FROM ( + SELECT /*+leading(ta broadcast tb)*/ 1 AS k, ta.b + tb.b AS z + FROM t1 ta JOIN t1 tb ON ta.a = tb.a + WHERE ta.a = 999 + ) sub, t2 + GROUP BY k; + """ + + order_qt_const_group_key_exe """ + SELECT k, SUM(z) + FROM ( + SELECT /*+leading(ta broadcast tb)*/ 1 AS k, ta.b + tb.b AS z + FROM t1 ta JOIN t1 tb ON ta.a = tb.a + WHERE ta.a = 999 + ) sub, t2 + GROUP BY k; + """ + + // Test: valid pushdown must be preserved. Same shape as above, but + // z = ta.b comes from one side only. The context reaching the Project + // still ends up with groupKeys=[] (k is a constant), yet at the inner + // join fillGroupByKeys adds the join key ta.a, so the aggregate finally + // pushed onto scan(ta) is a grouped one: agg(group=[a], sum(b)). + // This is why the empty-group-keys guard must sit at genAggregate (the + // aggregate-creation boundary) instead of firing early when an + // intermediate context has no group keys. + + // expect an aggregate pushed below the inner join, on top of scan(ta) + qt_pushdown_preserved_shape """ + explain shape plan + SELECT k, SUM(z) + FROM ( + SELECT /*+leading(ta broadcast tb)*/ 1 AS k, ta.b AS z + FROM t1 ta JOIN t1 tb ON ta.a = tb.a + ) sub, t2 + GROUP BY k; + """ + + // (10 + 20) * 2 rows of t2 = 60 + order_qt_pushdown_preserved_exe """ + SELECT k, SUM(z) + FROM ( + SELECT /*+leading(ta broadcast tb)*/ 1 AS k, ta.b AS z + FROM t1 ta JOIN t1 tb ON ta.a = tb.a + ) sub, t2 + GROUP BY k; + """ +} diff --git a/regression-test/suites/query_p0/expression/fold_constant/fe_constant_cast_to_date.groovy b/regression-test/suites/query_p0/expression/fold_constant/fe_constant_cast_to_date.groovy index 933ac7a47fd18e..87c2ebdc34d9e0 100644 --- a/regression-test/suites/query_p0/expression/fold_constant/fe_constant_cast_to_date.groovy +++ b/regression-test/suites/query_p0/expression/fold_constant/fe_constant_cast_to_date.groovy @@ -57,36 +57,36 @@ suite("fe_constant_cast_to_date") { test { sql """select cast("2023-07-16T19.123+08:00" as date)""" - exception "can't cast to DATETIMEV2" + exception "can't cast to DATEV2" } qt_sql """select cast("2024/05/01" as date)""" test { sql """select cast("24012" as date)""" - exception "can't cast to DATETIMEV2" + exception "can't cast to DATEV2" } test { sql """select cast("2411 123" as date)""" - exception "can't cast to DATETIMEV2" + exception "can't cast to DATEV2" } test { sql """select cast("2024-05-01 01:030:02" as date)""" - exception "can't cast to DATETIMEV2" + exception "can't cast to DATEV2" } test { sql """select cast("10000-01-01 00:00:00" as date)""" - exception "can't cast to DATETIMEV2" + exception "can't cast to DATEV2" } test { sql """select cast("2024-0131T12:00" as date)""" - exception "can't cast to DATETIMEV2" + exception "can't cast to DATEV2" } test { sql """select cast("2024-05-01@00:00" as date)""" - exception "can't cast to DATETIMEV2" + exception "can't cast to DATEV2" } test { sql """select cast("20120212051" as date)""" - exception "can't cast to DATETIMEV2" + exception "can't cast to DATEV2" } test { sql """select cast("2024-05-01T00:00XYZ" as date)""" @@ -114,7 +114,7 @@ suite("fe_constant_cast_to_date") { } test { sql """select cast("2024-05-01T00:00+08:25" as date)""" - exception "can't cast to DATETIMEV2" + exception "can't cast to DATEV2" } test { sql """select cast(1000 as date)""" diff --git a/regression-test/suites/query_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy b/regression-test/suites/query_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy index 522636aef464d9..6963e2363d52c9 100644 --- a/regression-test/suites/query_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy +++ b/regression-test/suites/query_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy @@ -155,6 +155,15 @@ suite("fold_constant_string_arithmatic") { testFoldConst("select field('=', '+', '=', '=', 'こ')") testFoldConst("select field('==', '+', '=', '==', 'こ')") testFoldConst("select field('=', '+', '==', '==', 'こ')") + testFoldConst("select field(cast('nan' as float), cast('nan' as float)), " + + "field(cast('nan' as double), cast('nan' as double))") + testFoldConst("select field(cast(0.0 as float), cast(-0.0 as float)), " + + "field(cast(-0.0 as float), cast(0.0 as float)), " + + "field(cast(0.0 as double), cast(-0.0 as double)), " + + "field(cast(-0.0 as double), cast(0.0 as double))") + + // cast decimalv3 to string + testFoldConst("select cast(cast('1E+3' as decimalv3(10, 2)) as string)") // find_in_set testFoldConst("select find_in_set('a', null)") @@ -2048,4 +2057,3 @@ suite("fold_constant_string_arithmatic") { testFoldConst("SELECT IS_UUID('6ccd780cbaba102')") testFoldConst("SELECT IS_UUID(NULL)") } - diff --git a/regression-test/suites/query_p0/join/test_join5.groovy b/regression-test/suites/query_p0/join/test_join5.groovy index 4323575870ffc4..fe2f629ca869b0 100644 --- a/regression-test/suites/query_p0/join/test_join5.groovy +++ b/regression-test/suites/query_p0/join/test_join5.groovy @@ -101,6 +101,8 @@ suite("test_join5", "query,p0") { where f2 = 53; """ + // Avoid exposing a dangling view to concurrent information_schema scans. + sql "DROP VIEW IF EXISTS ${tbName7};" sql "DROP TABLE IF EXISTS ${tbName4};" sql "DROP TABLE IF EXISTS ${tbName5};" sql "DROP TABLE IF EXISTS ${tbName6};" diff --git a/regression-test/suites/query_p0/runtime_filter/rf_partition_pruning.groovy b/regression-test/suites/query_p0/runtime_filter/rf_partition_pruning.groovy index 1b76f964aac7b2..4776c967ceaf5b 100644 --- a/regression-test/suites/query_p0/runtime_filter/rf_partition_pruning.groovy +++ b/regression-test/suites/query_p0/runtime_filter/rf_partition_pruning.groovy @@ -1479,6 +1479,59 @@ suite("rf_partition_pruning", "nonConcurrent") { + "ON f.region_id + f.region_id = d.dim_region", "BLOOM_FILTER", 5, 3) + // Test 50b: NoneMovable target expressions must not be evaluated speculatively + // against LIST partition values. The part_key=0 row is removed by keep_row=1 + // before the join, so assert_true must not fail only because RF partition pruning + // evaluates the target expression on the p_zero boundary. + sql "drop table if exists rf_prune_expr_fact" + sql """ + CREATE TABLE rf_prune_expr_fact ( + id BIGINT NOT NULL, + part_key INT NOT NULL, + keep_row INT NOT NULL + ) DUPLICATE KEY(id) + PARTITION BY LIST(part_key) ( + PARTITION p_zero VALUES IN (0), + PARTITION p_one VALUES IN (1) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + sql "INSERT INTO rf_prune_expr_fact VALUES (0, 0, 0), (1, 1, 1)" + + sql "drop table if exists rf_prune_expr_dim" + sql """ + CREATE TABLE rf_prune_expr_dim ( + flag BOOLEAN NOT NULL + ) DUPLICATE KEY(flag) + DISTRIBUTED BY HASH(flag) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + sql "INSERT INTO rf_prune_expr_dim VALUES (true)" + + sql "set enable_runtime_filter_partition_prune=false" + qt_rf_prune_expr_without_partition_prune """ + SELECT /*+ SET_VAR(runtime_filter_type='IN') */ COUNT(*) + FROM rf_prune_expr_fact fact + JOIN [broadcast] rf_prune_expr_dim dim + ON assert_true(fact.part_key != 0, 'rfpp_expr_in_only_error') = dim.flag + WHERE fact.keep_row = 1 + """ + + sql "set enable_runtime_filter_partition_prune=true" + qt_rf_prune_expr_with_partition_prune """ + SELECT /*+ SET_VAR(runtime_filter_type='IN') */ COUNT(*) + FROM rf_prune_expr_fact fact + JOIN [broadcast] rf_prune_expr_dim dim + ON assert_true(fact.part_key != 0, 'rfpp_expr_in_only_error') = dim.flag + WHERE fact.keep_row = 1 + """ + assertNoPartitionPruningProfile( + "COUNT(*) FROM rf_prune_expr_fact fact JOIN [broadcast] rf_prune_expr_dim dim " + + "ON assert_true(fact.part_key != 0, 'rfpp_expr_in_only_error') = dim.flag " + + "WHERE fact.keep_row = 1", + "IN") + // ============================================================ // Test 51: String partition column (LIST partition on VARCHAR). // diff --git a/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy b/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy index 5533853eaa93ad..ee118bccbd5730 100644 --- a/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy +++ b/regression-test/suites/query_p0/set_operations/bucket_shuffle_set_operation.groovy @@ -16,9 +16,6 @@ // under the License. suite("bucket_shuffle_set_operation") { - // TODO: open comment when support `enable_local_shuffle_planner` and change to REQUIRE - return - multi_sql """ drop table if exists bucket_shuffle_set_operation1; create table bucket_shuffle_set_operation1(id int, value int) distributed by hash(id) buckets 10 properties('replication_num'='1'); @@ -37,6 +34,9 @@ suite("bucket_shuffle_set_operation") { // make bucket shuffle set operation stable sql "set parallel_pipeline_task_num=5" + // disable the bucket shuffle downgrade so the chosen shapes do not depend on the + // backend count / parallelism of the environment running this suite + sql "set bucket_shuffle_downgrade_ratio=0" def checkShapeAndResult = { String tag, String sqlStr -> quickTest(tag + "_shape", "explain shape plan " + sqlStr) @@ -95,6 +95,315 @@ suite("bucket_shuffle_set_operation") { select id from bucket_shuffle_set_operation2 where id=1 """) + // The basic child of a bucket-shuffle set operation can be a join output instead of a + // direct scan. In that shape the local exchange planned for the basic side must still + // partition by the storage bucket function: an execution-hash local exchange would not + // align with the bucket-distributed side and the set operation would compute wrong results. + checkShapeAndResult("bucket_shuffle_join_as_basic_child", """ + select a.id from bucket_shuffle_set_operation1 a + join bucket_shuffle_set_operation2 b on a.id = b.id + intersect + select id from bucket_shuffle_set_operation3""") + + // a set operation child can itself be a set operation whose output claims a bucket + // distribution; the outer set operation must only treat its children as bucket-aligned + // when they share the same storage layout + checkShapeAndResult("bucket_shuffle_nested_set_operation", """ + select id from bucket_shuffle_set_operation3 + union all + (select a.id from bucket_shuffle_set_operation1 a + join bucket_shuffle_set_operation2 b on a.id = b.id + intersect + select id from bucket_shuffle_set_operation2)""") + + // when local shuffle is disabled entirely, every pipeline runs a single task per + // instance so the bucket alignment holds naturally and bucket shuffle is still allowed + sql "set enable_local_shuffle=false" + checkShapeAndResult("bucket_shuffle_when_local_shuffle_off", """ + select id from bucket_shuffle_set_operation1 + intersect + select id from bucket_shuffle_set_operation2""") + sql "set enable_local_shuffle=true" + + // A shuffle join above the union pushes a hash request into the union + // (createHashRequestAccordingToParent, the parent-hash request path). When the FE does not + // plan the local shuffle, that request must be downgraded so the union does not choose + // bucket shuffle, while the result stays correct. + def unionParentHashSql = """ + select b.id from ( + select id from bucket_shuffle_set_operation1 + union all + select id from bucket_shuffle_set_operation2 + ) u join[shuffle] bucket_shuffle_set_operation3 b on u.id = b.id + """ + sql "set enable_local_shuffle_planner=false" + // Golden shape: the union must stay a plain PhysicalUnion whose two children are each a + // PhysicalDistribute[DistributionSpecHash]. That is the actual proof that the parent hash + // request was pushed down into the union (createHashRequestAccordingToParent) and downgraded + // to execution hash, not merely that the PhysicalUnion line lacks the [bucketShuffle] tag. + // If that path regressed, the optimizer could instead keep an unbucketed union and add a + // single PhysicalDistribute[DistributionSpecHash] above it for the shuffle join; the golden + // shape below (union children are distributes, not direct scans) would catch that. + qt_union_parent_hash_shape_when_local_shuffle_planner_off("explain shape plan " + unionParentHashSql) + explain { + sql "shape plan " + unionParentHashSql + check { String e -> + def unionIndex = e.indexOf("PhysicalUnion") + assertTrue(unionIndex >= 0) + // the union must not be a bucket shuffle union when the FE local shuffle planner is off + assertFalse(e.substring(unionIndex, + Math.min(unionIndex + "PhysicalUnion".length() + 20, e.length())).contains("bucketShuffle")) + // and the parent hash request must have been pushed into the union: each union child + // arrives through a PhysicalDistribute[DistributionSpecHash], so no union child is a + // direct scan. + def afterUnion = e.substring(unionIndex) + def joinProbeIndex = afterUnion.indexOf("bucket_shuffle_set_operation3") + def unionSubtree = joinProbeIndex >= 0 ? afterUnion.substring(0, joinProbeIndex) : afterUnion + assertEquals(2, unionSubtree.split("PhysicalDistribute\\[DistributionSpecHash\\]", -1).length - 1) + } + } + order_qt_union_parent_hash_when_local_shuffle_planner_off unionParentHashSql + sql "set enable_local_shuffle_planner=true" + + // A plain intersect/except without a parent hash request goes through + // visitPhysicalSetOperation directly (not the parent-hash request path); with the FE local + // shuffle planner disabled it must not choose bucket shuffle either, and the result stays + // correct. + sql "set enable_local_shuffle_planner=false" + explain { + sql "shape plan select id from bucket_shuffle_set_operation1 intersect select id from bucket_shuffle_set_operation2" + check { String e -> + assertFalse(e.contains("bucketShuffle")) + } + } + order_qt_plain_intersect_when_local_shuffle_planner_off "select id from bucket_shuffle_set_operation1 intersect select id from bucket_shuffle_set_operation2" + sql "set enable_local_shuffle_planner=true" + + // Set operation bucket shuffle is gated on setOperationBucketShuffleAllowed() = + // enableLocalShufflePlanner && canUseNereidsDistributePlanner. The enable_local_shuffle_planner + // =false cases above only exercise the first conjunct; this block guards the second one. With + // local shuffle and the FE local-shuffle planner both enabled but the Nereids distribute planner + // disabled, the set operation must still downgrade (no [bucketShuffle]) and keep correct results. + sql "set enable_local_shuffle=true" + sql "set enable_local_shuffle_planner=true" + sql "set enable_nereids_distribute_planner=false" + // plain intersect goes through visitPhysicalSetOperation directly + explain { + sql "shape plan select id from bucket_shuffle_set_operation1 intersect select id from bucket_shuffle_set_operation2" + check { String e -> + assertFalse(e.contains("bucketShuffle")) + } + } + order_qt_plain_intersect_when_nereids_distribute_planner_off "select id from bucket_shuffle_set_operation1 intersect select id from bucket_shuffle_set_operation2" + // the parent-hash union path (createHashRequestAccordingToParent) must downgrade too + explain { + sql "shape plan " + unionParentHashSql + check { String e -> + def unionIndex = e.indexOf("PhysicalUnion") + assertTrue(unionIndex >= 0) + assertFalse(e.substring(unionIndex, + Math.min(unionIndex + "PhysicalUnion".length() + 20, e.length())).contains("bucketShuffle")) + } + } + order_qt_union_parent_hash_when_nereids_distribute_planner_off unionParentHashSql + sql "set enable_nereids_distribute_planner=true" + + // The right child can be selected as the bucket-shuffle basic child (larger row count). The + // set operation output must then advertise a non-specific distribution rather than a plain + // execution hash: otherwise two such set operations with different storage layouts are + // co-located under a join and fail bucket assignment ("Can not find tablet ... in the + // bucket"). r1 / r2 are larger than the small tables so they become the basic child on the + // right, and they are different tables so their bucket layouts differ. + sql "drop table if exists bucket_shuffle_set_operation_r1" + sql "create table bucket_shuffle_set_operation_r1(id int) distributed by hash(id) buckets 10 properties('replication_num'='1')" + sql "insert into bucket_shuffle_set_operation_r1 select number from numbers('number'='20')" + sql "drop table if exists bucket_shuffle_set_operation_r2" + sql "create table bucket_shuffle_set_operation_r2(id int) distributed by hash(id) buckets 10 properties('replication_num'='1')" + sql "insert into bucket_shuffle_set_operation_r2 select number from numbers('number'='20')" + sql "alter table bucket_shuffle_set_operation_r1 modify column id set stats ('row_count'='1000', 'ndv'='1000', 'min_value'='0', 'max_value'='19')" + sql "alter table bucket_shuffle_set_operation_r2 modify column id set stats ('row_count'='1000', 'ndv'='1000', 'min_value'='0', 'max_value'='19')" + def intersectRightBasicSql = """ + select t.id from + (select id from bucket_shuffle_set_operation1 intersect select id from bucket_shuffle_set_operation_r1) t + join[shuffle] + (select id from bucket_shuffle_set_operation1 intersect select id from bucket_shuffle_set_operation_r2) s + on t.id = s.id""" + // Prove the shuffleToRight branch (distributeToChildIndex > 0) is actually covered, under a + // parent hash consumer. The join[shuffle] hint forces the parent to be a hash consumer of the + // two set-operation outputs. Each INTERSECT must keep the right table (r1 / r2, the larger row + // count) as the direct bucketed basic child while the left op1 is the enforced + // PhysicalDistribute[DistributionSpecHash] side. Because visitPhysicalSetOperation keeps the + // right basic child's specific storage layout, the two intersects carry different table ids + // (r1 vs r2), so the parent re-aligns them with a bucket-shuffle join instead of co-locating + // them (which, if the layout were erased to a layout-less execution hash, would fail bucket + // assignment with "Can not find tablet"). If the right side stopped being selected as basic, or + // the layout were dropped so the join co-located, these assertions fail even though the result + // stays 1, 2, 3. + explain { + sql "shape plan " + intersectRightBasicSql + check { String e -> + assertTrue(e.contains("PhysicalIntersect[bucketShuffle]")) + // the parent join re-aligns the two different-layout outputs (bucket-shuffle hash + // consumer) and does not co-locate them + assertTrue(e.contains("hashJoin[INNER_JOIN bucketShuffle]"), + "parent must re-align the two different-layout outputs with a bucket-shuffle join") + assertFalse(e.contains("colocated"), "parent must not co-locate the two right-basic intersects") + def lines = e.split("\n").findAll { it =~ /Physical|hashJoin/ } + def depth = { String s -> (s =~ /^-*/)[0].length() } + def parentOf = { int idx -> + for (int k = idx - 1; k >= 0; k--) { + if (depth(lines[k]) < depth(lines[idx])) { + return lines[k] + } + } + return "" + } + // each INTERSECT keeps r1 / r2 (the larger, right side) as its direct bucketed basic child + ["bucket_shuffle_set_operation_r1", "bucket_shuffle_set_operation_r2"].each { rt -> + int i = lines.findIndexOf { it.contains("PhysicalOlapScan[" + rt + "]") } + assertTrue(i >= 0, rt + " scan not found in shape") + assertTrue(parentOf(i).contains("PhysicalIntersect[bucketShuffle]"), + rt + " must be the direct bucketed basic child of the intersect (right basic)") + } + // and each INTERSECT's other (left) child arrives through an enforced + // PhysicalDistribute[DistributionSpecHash] directly under the intersect + int enforcedLeftCount = 0 + lines.eachWithIndex { String ln, int i -> + if (ln.contains("PhysicalDistribute[DistributionSpecHash]") + && parentOf(i).contains("PhysicalIntersect[bucketShuffle]")) { + enforcedLeftCount++ + } + } + assertTrue(enforcedLeftCount >= 2, + "each intersect must shuffle its left child through a PhysicalDistribute[DistributionSpecHash]") + } + } + order_qt_intersect_right_basic_parent_hash intersectRightBasicSql + + // A non-intersect bucket-shuffle set operation (UNION ALL) whose basic / anchor child is a + // direct bucketed scan that is bucket-pruned by an IN predicate on the distribution key, so + // it only scans a subset of the buckets. The other child is shuffled onto the anchor's + // storage layout and has rows in the buckets the pruned anchor does not scan. Because the + // union is a non-intersect bucket-shuffle set operation, UnassignedScanBucketOlapTableJob + // must fill up receiver instances for those missing buckets; without the fill-up the other + // child's rows in the missing buckets would have no destination instance and be lost. + // + // The setup forces the pruned scan to be the anchor: fill_anchor has huge injected stats so + // the IN-filtered branch still wins the largest-row-count basic-child selection, while + // fill_spread has a mismatched bucket count (11 vs 10) so it cannot be the natural anchor and + // must be shuffled. The bucket-shuffle join above the union supplies the parent hash request + // that makes the union choose bucket shuffle, and the join probe fill_probe has yet another + // bucket count (7) so the join is a real shuffle in its own fragment and does not co-locate a + // full-bucket scan into the union fragment (which would otherwise cover the missing buckets + // and hide the fill-up). + sql "drop table if exists bucket_shuffle_set_operation_fill_anchor" + sql """create table bucket_shuffle_set_operation_fill_anchor(id int) + distributed by hash(id) buckets 10 properties('replication_num'='1')""" + sql "insert into bucket_shuffle_set_operation_fill_anchor select number from numbers('number'='20')" + sql "drop table if exists bucket_shuffle_set_operation_fill_spread" + sql """create table bucket_shuffle_set_operation_fill_spread(id int) + distributed by hash(id) buckets 11 properties('replication_num'='1')""" + sql "insert into bucket_shuffle_set_operation_fill_spread select number from numbers('number'='20')" + sql "drop table if exists bucket_shuffle_set_operation_fill_probe" + sql """create table bucket_shuffle_set_operation_fill_probe(id int) + distributed by hash(id) buckets 7 properties('replication_num'='1')""" + sql "insert into bucket_shuffle_set_operation_fill_probe select number from numbers('number'='20')" + sql """alter table bucket_shuffle_set_operation_fill_anchor modify column id + set stats ('row_count'='1000000', 'ndv'='20', 'min_value'='0', 'max_value'='19')""" + sql """alter table bucket_shuffle_set_operation_fill_spread modify column id + set stats ('row_count'='50', 'ndv'='20', 'min_value'='0', 'max_value'='19')""" + sql """alter table bucket_shuffle_set_operation_fill_probe modify column id + set stats ('row_count'='30', 'ndv'='20', 'min_value'='0', 'max_value'='19')""" + def bucketShuffleUnionFillUpSql = """ + select t.id from ( + select id from bucket_shuffle_set_operation_fill_anchor where id in (0, 2, 4, 6) + union all + select id from bucket_shuffle_set_operation_fill_spread + ) t join[shuffle] bucket_shuffle_set_operation_fill_probe c on t.id = c.id""" + explain { + sql "shape plan " + bucketShuffleUnionFillUpSql + check { String e -> + assertTrue(e.contains("PhysicalUnion[bucketShuffle]")) + } + } + order_qt_bucket_shuffle_union_fill_up bucketShuffleUnionFillUpSql + + // Same missing-bucket fill-up contract, but the pruned basic child exposes its storage bucket + // key only through an equivalent slot: the union's basic child is a bucket-shuffle join output + // that projects the join key bucket_shuffle_set_operation2.id AS k, so the storage bucket column + // (fill_anchor.id) is hidden and only the equivalent k is visible in the set-operation output. + // The alignment proof must resolve the bucket key through its hash equivalence set (mirroring + // ChildrenPropertiesRegulator.canMapBucketKeysToRequire); a direct ExprId lookup would report + // the union as not bucket-aligned, drop the BUCKET_SHUFFLE marker, and skip + // UnassignedScanBucketOlapTableJob.fillUpInstances(), losing the shuffled side's rows in the + // buckets the pruned basic child does not scan. + def bucketShuffleEquivalentKeyFillUpSql = """ + select t.k from ( + select b.id as k from bucket_shuffle_set_operation_fill_anchor a + join[shuffle] bucket_shuffle_set_operation2 b on a.id = b.id + where a.id in (0, 2, 4, 6) + union all + select id from bucket_shuffle_set_operation_fill_spread + ) t join[shuffle] bucket_shuffle_set_operation_fill_probe c on t.k = c.id""" + explain { + sql "shape plan " + bucketShuffleEquivalentKeyFillUpSql + check { String e -> + assertTrue(e.contains("PhysicalUnion[bucketShuffle]")) + } + } + order_qt_bucket_shuffle_equivalent_key_fill_up bucketShuffleEquivalentKeyFillUpSql + + // Same shape but the hidden bucket key has two visible equivalent set-output columns: the join + // chain a.id = b.id = c.id makes fill_anchor.id equivalent to both projected columns x (c.id) + // and y (b.id), and the parent hashes the union output on the later one (y). The bucket key + // alignment must resolve to the same output position across children (the enforced sibling is + // shuffled by y), which is why the proof intersects each child's candidate equivalent positions + // rather than letting the basic child pick its first visible equivalent (x) and disagree with + // the sibling on y. + def bucketShuffleTwoEquivalentKeysSql = """ + select t.x, t.y from ( + select c.id as x, b.id as y from bucket_shuffle_set_operation_fill_anchor a + join[shuffle] bucket_shuffle_set_operation2 b on a.id = b.id + join[shuffle] bucket_shuffle_set_operation3 c on a.id = c.id + where a.id in (0, 2, 4, 6) + union all + select id, id from bucket_shuffle_set_operation_fill_spread + ) t join[shuffle] bucket_shuffle_set_operation_fill_probe p on t.y = p.id""" + explain { + sql "shape plan " + bucketShuffleTwoEquivalentKeysSql + check { String e -> + assertTrue(e.contains("PhysicalUnion[bucketShuffle]")) + } + } + order_qt_bucket_shuffle_two_equivalent_keys bucketShuffleTwoEquivalentKeysSql + + // The basic child candidate can be bucketed by MORE columns than the set operation distributes + // on: a table distributed by hash(k, v) feeding an INTERSECT on only k. The (k, v) bucket key is + // wider than the single-column requirement, so canMapBucketKeysToRequire() must reject it as the + // bucket-shuffle basic and the set operation falls back to execution-hash shuffle. Without that + // guard the planner hits a checkState in calAnotherSideRequiredShuffleIds and fails to plan. Two + // such tables (different bucket counts so neither is a natural colocate basic) force the fallback + // on both sides. + sql "drop table if exists bucket_shuffle_set_operation_kv" + sql """create table bucket_shuffle_set_operation_kv(k int, v int) + distributed by hash(k, v) buckets 10 properties('replication_num'='1')""" + sql "insert into bucket_shuffle_set_operation_kv values (1, 1), (2, 2), (3, 3)" + sql "drop table if exists bucket_shuffle_set_operation_kv2" + sql """create table bucket_shuffle_set_operation_kv2(k int, v int) + distributed by hash(k, v) buckets 11 properties('replication_num'='1')""" + sql "insert into bucket_shuffle_set_operation_kv2 values (1, 1), (2, 2), (3, 3)" + def multiColumnBucketKeyIntersectSql = """ + select k from bucket_shuffle_set_operation_kv + intersect + select k from bucket_shuffle_set_operation_kv2""" + explain { + sql "shape plan " + multiColumnBucketKeyIntersectSql + check { String e -> + assertFalse(e.contains("bucketShuffle")) + } + } + order_qt_multi_column_bucket_key_intersect multiColumnBucketKeyIntersectSql + explain { sql """ select id, id as id2 from (select nullable(id) as id from bucket_shuffle_set_operation1)a diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_of_array_difference.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_of_array_difference.groovy index 19db81787292a7..e0f4beff26da53 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_of_array_difference.groovy +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions_of_array_difference.groovy @@ -38,11 +38,17 @@ suite("test_array_functions_of_array_difference") { sql """ INSERT INTO ${tableName} VALUES(5, [16,7,8]) """ sql """ INSERT INTO ${tableName} VALUES(6, [1,2,3,4,5,4,3,2,1]) """ sql """ INSERT INTO ${tableName} VALUES(7, [1111,12324,8674,123,3434,435,45,53,54,2]) """ + sql """ INSERT INTO ${tableName} VALUES(8, [42]) """ + sql """ INSERT INTO ${tableName} VALUES(9, [2147483647,-2147483648]) """ + sql """ INSERT INTO ${tableName} VALUES(10, [-2147483648,2147483647]) """ qt_select "SELECT *, array_difference(k2) FROM ${tableName} order by k1" // literal qt_select "SELECT array_difference([1, 2, 3, 4]);" qt_select "SELECT array_difference([1, 7, 100, 5]);" + qt_select "SELECT array_difference([42]);" + qt_select "SELECT array_difference([2147483647, -2147483648]);" + qt_select "SELECT array_difference([-2147483648, 2147483647]);" } diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_nested_array_map.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_nested_array_map.groovy new file mode 100644 index 00000000000000..7741ce47ff2ca3 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_nested_array_map.groovy @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_nested_array_map") { + sql "DROP TABLE IF EXISTS test_nested_array_map_insert_src" + sql "DROP TABLE IF EXISTS test_nested_array_map_insert_dst" + + sql """ + CREATE TABLE test_nested_array_map_insert_src ( + id INT, + bucket_counts ARRAY + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + CREATE TABLE test_nested_array_map_insert_dst ( + id INT, + bucket_counts ARRAY + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + INSERT INTO test_nested_array_map_insert_src VALUES + (1, [1, 2, 3]), + (1, [4, 5, 6]), + (2, [10, 20]), + (2, [1, 2, 3]) + """ + + sql """ + INSERT INTO test_nested_array_map_insert_dst (id, bucket_counts) + WITH rollup_grouped AS ( + SELECT + id, + ARRAY_AGG(bucket_counts) AS bucket_count_arrays, + MAX(ARRAY_SIZE(bucket_counts)) AS max_bucket_len + FROM test_nested_array_map_insert_src + GROUP BY id + ) + SELECT + id, + ARRAY_MAP( + i -> ARRAY_SUM(ARRAY_MAP(a -> COALESCE(a[CAST(i AS INT)], 0), bucket_count_arrays)), + ARRAY_RANGE(1, max_bucket_len + 1) + ) AS bucket_counts + FROM rollup_grouped + """ + + order_qt_select """ + SELECT id, bucket_counts + FROM test_nested_array_map_insert_dst + ORDER BY id + """ + + qt_select2 """ + select array_map(x -> array_map(y -> y - x, [1, 2]), [10, 20]); + """ + + qt_select_nested_array_sortby """ + select array_map(x -> array_sortby(y -> abs(y - x), [5, 1, 3, 9]), [2, 6, 8]); + """ + + qt_select_same_name_shadow """ + select array_map(x -> array_map(x -> x + 1, x), [[1, 2], [3, 4]]); + """ +} diff --git a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy index fabfef875d31ba..2c6640a65158c1 100644 --- a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy @@ -16,6 +16,12 @@ // under the License. suite("test_bitmap_function") { + def low_bitmap = "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32" + def overlap_bitmap = "32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64" + def disjoint_bitmap = "33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65" + def high_bitmap = "4294967296,4294967297,4294967298,4294967299,4294967300,4294967301,4294967302,4294967303,4294967304,4294967305,4294967306,4294967307,4294967308,4294967309,4294967310,4294967311,4294967312,4294967313,4294967314,4294967315,4294967316,4294967317,4294967318,4294967319,4294967320,4294967321,4294967322,4294967323,4294967324,4294967325,4294967326,4294967327,4294967328" + def two_map_bitmap = "${low_bitmap},${high_bitmap}" + // BITMAP_AND qt_sql """ select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(2))) cnt """ qt_sql """ select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(1))) cnt """ @@ -39,10 +45,17 @@ suite("test_bitmap_function") { // BITMAP_HAS_ANY qt_sql """ select bitmap_has_any(to_bitmap(1),to_bitmap(2)) cnt """ qt_sql """ select bitmap_has_any(to_bitmap(1),to_bitmap(1)) cnt """ + qt_sql """ select bitmap_has_any(bitmap_from_string('${low_bitmap}'), bitmap_from_string('${overlap_bitmap}')) cnt """ + qt_sql """ select bitmap_has_any(bitmap_from_string('${low_bitmap}'), bitmap_from_string('${disjoint_bitmap}')) cnt """ + qt_sql """ select bitmap_has_any(bitmap_from_string('${low_bitmap}'), bitmap_from_string('${high_bitmap}')) cnt """ + qt_sql """ select bitmap_has_any(bitmap_from_string('${two_map_bitmap}'), bitmap_from_string('${low_bitmap}')) cnt """ + qt_sql """ select bitmap_has_any(bitmap_from_string('${low_bitmap}'), bitmap_from_string('${two_map_bitmap}')) cnt """ // BITMAP_HAS_ALL qt_sql """ select bitmap_has_all(bitmap_from_string("0, 1, 2"), bitmap_from_string("1, 2")) cnt """ qt_sql """ select bitmap_has_all(bitmap_empty(), bitmap_from_string("1, 2")) cnt """ + qt_sql """ select bitmap_has_all(bitmap_from_string('${low_bitmap}'), bitmap_from_string('${low_bitmap}')) cnt """ + qt_sql """ select bitmap_has_all(bitmap_from_string('${low_bitmap}'), bitmap_from_string('${overlap_bitmap}')) cnt """ // BITMAP_HASH qt_sql_bitmap_hash1 """ select bitmap_count(bitmap_hash('hello')) """ diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy index e6780eb888c47f..a523b16656c9e3 100644 --- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy @@ -496,6 +496,7 @@ suite("test_date_function") { check_fold_consistency("str_to_date('2026-01-28 11:32:47.1234567', '%Y-%m-%d %T.%f')") check_fold_consistency("str_to_date('2026-01-28 11:32:47.123456789', '%Y-%m-%d %T.%f')") check_fold_consistency("str_to_date('2026-01-28 11:32:47', '%Y-%m-%d %T')") + testFoldConst("select str_to_date('2024-01-01', concat('%Y-%m', '-%d'))") sql """ truncate table ${tableName} """ sql """ insert into ${tableName} values ("2020-09-01") """ qt_sql """ select str_to_date(test_datetime, "%Y-%m-%d %H:%i:%s") from ${tableName};""" diff --git a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy index eec99e217824d7..baed2ba4324cc5 100644 --- a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy @@ -83,10 +83,11 @@ suite("test_json_function", "arrow_flight_sql") { qt_sql "SELECT json_extract_no_quotes('[1, 2, 3]', '\$.[1]');" qt_sql "SELECT json_extract_no_quotes('{\"id\": 123, \"name\": \"doris\"}', '\$.name');" - qt_sql "SELECT json_extract_no_quotes('{\"id\": 123, \"name\": \"doris\"}', '\$.id', null);" qt_sql "SELECT json_extract_no_quotes(null, '\$.id');" - qt_sql "SELECT json_extract_no_quotes('{\"k1\": \"v1\", \"k2\": { \"k21\": 6.6, \"k22\": [1, 2, 3] } }', '\$.k1', '\$.k2');" - qt_sql "SELECT json_extract_no_quotes('{\"k1\": \"v1\", \"k2\": { \"k21\": 6.6, \"k22\": [1, 2, 3] } }', '\$.k2.k21', '\$.k2.k22', '\$.k2.k22[1]');" + test { + sql "SELECT json_extract_no_quotes('{\"id\": 123, \"name\": \"doris\"}', '\$.id', null);" + exception "Can not found function 'json_extract_no_quotes' which has 3 arity" + } // invalid json path qt_sql """select get_json_string('{"name\\k" : 123}', '\$.name\\k')""" @@ -317,4 +318,4 @@ suite("test_json_function", "arrow_flight_sql") { """ sql "drop table if exists json_remove_test_table;" -} \ No newline at end of file +} diff --git a/regression-test/suites/query_p0/stats/query_stats_test.groovy b/regression-test/suites/query_p0/stats/query_stats_test.groovy index 9a445e88e27361..0cadec47cc2e9a 100644 --- a/regression-test/suites/query_p0/stats/query_stats_test.groovy +++ b/regression-test/suites/query_p0/stats/query_stats_test.groovy @@ -198,6 +198,78 @@ suite("query_stats_test") { def joinResult = sql "show query stats from ${tbName} all" assertTrue((joinResult[0][1] as int) >= 1) + // UNION ALL: both branches contribute queryHit for k1; WHERE on right branch adds k2.filterHit. + sql "clean all query stats" + sql "select k1 from ${tbName} union all select k1 from ${tbName} where k2 = 100" + def unionStats = sql "show query stats from ${tbName}" + def uK1 = unionStats.find { it[0] == "k1" } + def uK2 = unionStats.find { it[0] == "k2" } + assertNotNull(uK1) + assertNotNull(uK2) + assertTrue((uK1[1] as int) >= 1, "k1: UNION ALL queryHit") + assertTrue((uK2[2] as int) >= 1, "k2: WHERE on right UNION branch filterHit") + + // HAVING: k1 queryHit from GROUP BY + SELECT, k2 queryHit from SUM input, + // k2 filterHit from HAVING SUM(k2) > -1000. + sql "clean all query stats" + sql "select k1, sum(k2) from ${tbName} group by k1 having sum(k2) > -1000" + def havingStats = sql "show query stats from ${tbName}" + def hvK1 = havingStats.find { it[0] == "k1" } + def hvK2 = havingStats.find { it[0] == "k2" } + assertNotNull(hvK1) + assertNotNull(hvK2) + assertTrue((hvK1[1] as int) >= 1, "k1: GROUP BY / SELECT queryHit") + assertTrue((hvK2[1] as int) >= 1, "k2: SUM aggregate input queryHit") + assertTrue((hvK2[2] as int) >= 1, "k2: HAVING SUM(k2) filterHit") + + // CTE: consumer query records queryHit on k2 and filterHit on k1. + // Force materialization so the plan keeps PhysicalCTEProducer/Consumer instead of + // CTEInline collapsing the single-reference CTE into a plain scan+filter. + sql "clean all query stats" + sql "set inline_cte_referenced_threshold = 0" + sql """with cte as (select k2 from ${tbName} where k1 = 1) select k2 from cte""" + sql "set inline_cte_referenced_threshold = 1" + def cteStats = sql "show query stats from ${tbName}" + def cteK2 = cteStats.find { it[0] == "k2" } + def cteK1 = cteStats.find { it[0] == "k1" } + assertNotNull(cteK2) + assertNotNull(cteK1) + assertTrue((cteK2[1] as int) >= 1, "k2: CTE consumer queryHit") + assertTrue((cteK1[2] as int) >= 1, "k1: CTE WHERE filterHit") + + // LATERAL VIEW / EXPLODE: the generator input column (k7, a varchar) gets queryHit + // because it is the source expression fed into the generator; the synthetic output + // slot (e1) is skipped by the PhysicalGenerate handler. + sql "clean all query stats" + sql "select e1 from ${tbName} lateral view explode_split(k7, ',') tmp as e1" + def lateralStats = sql "show query stats from ${tbName}" + def lvK7 = lateralStats.find { it[0] == "k7" } + assertNotNull(lvK7, "k7 must appear after LATERAL VIEW EXPLODE_SPLIT") + assertTrue((lvK7[1] as int) >= 1, "k7: LATERAL VIEW input column queryHit") + + // Table-function join ON predicate: PhysicalGenerate.getConjuncts() is sent straight to + // TableFunctionNode and actually filters at execution — must record filterHit too. + // The predicate must reference a real column (k7), not just the generator's own + // synthetic output (val) — filtering on val alone has no scan column to attribute to. + sql "clean all query stats" + sql """select s.val from ${tbName} left join lateral unnest(split(k7, ',')) s(val) on k7 != ''""" + def genStats = sql "show query stats from ${tbName}" + def genK7 = genStats.find { it[0] == "k7" } + assertNotNull(genK7) + assertTrue((genK7[2] as int) >= 1, "k7: table-function join ON predicate filterHit") + + // Computed SELECT: SELECT k1+k2 — both input columns get queryHit even though + // the output expression is not a plain slot reference. + sql "clean all query stats" + sql "select k1 + k2 from ${tbName}" + def computedStats = sql "show query stats from ${tbName}" + def cmpK1 = computedStats.find { it[0] == "k1" } + def cmpK2 = computedStats.find { it[0] == "k2" } + assertNotNull(cmpK1, "k1 must appear in computed SELECT k1+k2") + assertNotNull(cmpK2, "k2 must appear in computed SELECT k1+k2") + assertTrue((cmpK1[1] as int) >= 1, "k1: computed SELECT queryHit") + assertTrue((cmpK2[1] as int) >= 1, "k2: computed SELECT queryHit") + sql "admin set all frontends config (\"enable_query_hit_stats\"=\"false\");" sql "set enable_nereids_planner = ${origNereids}" sql "set enable_query_cache = ${origCache}" diff --git a/regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy b/regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy index c1e40c33ba8a5a..d810420346ba79 100644 --- a/regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy +++ b/regression-test/suites/row_binlog_p0/test_row_binlog_basic.groovy @@ -293,6 +293,14 @@ suite("test_row_binlog_basic", "nonConcurrent") { FROM binlog("table" = "test_mow_seq_with_binlog") ORDER BY __DORIS_BINLOG_TSO__, __DORIS_BINLOG_LSN__ """ + + test { + sql """ + SELECT __DORIS_SEQUENCE_COL__ + FROM binlog("table" = "test_mow_seq_with_binlog") + """ + exception "Unknown column" + } sql "SET skip_delete_bitmap = false" -} \ No newline at end of file +} diff --git a/regression-test/suites/schema_change_p0/test_schema_change_with_empty_rowset.groovy b/regression-test/suites/schema_change_p0/test_schema_change_with_empty_rowset.groovy index 008b1033e9ef25..6a316b7e12948d 100644 --- a/regression-test/suites/schema_change_p0/test_schema_change_with_empty_rowset.groovy +++ b/regression-test/suites/schema_change_p0/test_schema_change_with_empty_rowset.groovy @@ -31,6 +31,42 @@ suite("test_schema_change_with_empty_rowset", "p0,nonConcurrent") { return jobStateResult[0][9] } + def backendIdToHost = [:] + def backendIdToHttpPort = [:] + getBackendIpHttpPort(backendIdToHost, backendIdToHttpPort) + + def triggerCumulativeAndWaitForVersionCount = { tbl, int targetVersionCount -> + def tablets = sql_return_maparray """ SHOW TABLETS FROM ${tbl} """ + for (def tablet in tablets) { + def host = backendIdToHost["${tablet.BackendId}"] + def port = backendIdToHttpPort["${tablet.BackendId}"] + def (code, out, err) = be_run_cumulative_compaction(host, port, tablet.TabletId) + assert code == 0: "trigger cumulative compaction failed, tablet=${tablet.TabletId}, " + + "code=${code}, stdout=${out}, stderr=${err}" + def triggerStatus = parseJson(out.trim()) + assert triggerStatus.status?.equalsIgnoreCase("Success"): + "trigger cumulative compaction failed, tablet=${tablet.TabletId}, stdout=${out}" + } + + // The rowsets returned by the BE are the active versions used by its load admission check. + Awaitility.await().atMost(300, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until { + def versionCounts = [:] + for (def tablet in tablets) { + def host = backendIdToHost["${tablet.BackendId}"] + def port = backendIdToHttpPort["${tablet.BackendId}"] + def (code, out, err) = be_show_tablet_status(host, port, tablet.TabletId) + assert code == 0: "get compaction status failed, tablet=${tablet.TabletId}, " + + "code=${code}, stdout=${out}, stderr=${err}" + def tabletStatus = parseJson(out.trim()) + assert tabletStatus.rowsets instanceof List: + "invalid compaction status, tablet=${tablet.TabletId}, stdout=${out}" + versionCounts[tablet.TabletId] = tabletStatus.rowsets.size() + } + logger.info("waiting for ${tbl} BE version count <= ${targetVersionCount}, current=${versionCounts}") + return versionCounts.values().every { it <= targetVersionCount } + } + } + sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ @@ -64,12 +100,14 @@ suite("test_schema_change_with_empty_rowset", "p0,nonConcurrent") { } - // trigger compactions for all tablets in ${tableName} - trigger_and_wait_compaction(tableName, "cumulative") + // Leave enough version headroom for the schema change and the following inserts. + int insertCountAfterCompaction = 20 + int targetVersionCount = custoBeConfig.max_tablet_version_num - insertCountAfterCompaction - 1 + triggerCumulativeAndWaitForVersionCount(tableName, targetVersionCount) sql """ alter table ${tableName} modify column k4 string NULL""" - for (int i = 100; i < 120; i++) { + for (int i = 100; i < 100 + insertCountAfterCompaction; i++) { sql """ insert into ${tableName} values ($i, 2, 3, 4, 5, 6.6, 1.7, 8.8, 'a', 'b', 'c', '2021-10-30', '2021-10-30 00:00:00') """ sleep(20) diff --git a/regression-test/suites/shape_check/tpcds_sf100/noStatsRfPrune/query64.groovy b/regression-test/suites/shape_check/tpcds_sf100/noStatsRfPrune/query64.groovy deleted file mode 100644 index a96231ec2fe306..00000000000000 --- a/regression-test/suites/shape_check/tpcds_sf100/noStatsRfPrune/query64.groovy +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -suite("query64") { - String db = context.config.getDbNameByFile(new File(context.file.parent)) - if (isCloudMode()) { - return - } - sql "use ${db}" - sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=false' - sql 'set enable_fallback_to_original_planner=false' - sql 'set exec_mem_limit=21G' - sql 'set be_number_for_test=3' - sql 'set enable_runtime_filter_prune=true' - sql 'set parallel_pipeline_task_num=8' - sql 'set forbid_unknown_col_stats=false' - sql 'set enable_stats=false' - sql "set runtime_filter_type=8" - sql 'set broadcast_row_count_limit = 30000000' - sql 'set enable_nereids_timeout = false' - sql 'SET enable_pipeline_engine = true' - sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - sql "set memo_max_group_expression_size = 1000000" - - def ds64 = ''' - with cs_ui as - (select cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), - cross_sales as - (select i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('blanched','medium','brown','chocolate','burlywood','drab') and - i_current_price between 23 and 23 + 10 and - i_current_price between 23 + 1 and 23 + 15 - group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year - ) - select cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt - from cross_sales cs1,cross_sales cs2 - where cs1.item_sk=cs2.item_sk and - cs1.syear = 2001 and - cs2.syear = 2001 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip - order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1; - - ''' - - qt_ds_shape_64 'explain shape plan ' + ds64 - -} diff --git a/regression-test/suites/shape_check/tpcds_sf100/no_stats_shape/query64.groovy b/regression-test/suites/shape_check/tpcds_sf100/no_stats_shape/query64.groovy deleted file mode 100644 index 28896677cf2ff1..00000000000000 --- a/regression-test/suites/shape_check/tpcds_sf100/no_stats_shape/query64.groovy +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -suite("query64") { - String db = context.config.getDbNameByFile(new File(context.file.parent)) - if (isCloudMode()) { - return - } - sql "use ${db}" - sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=false' - sql 'set enable_fallback_to_original_planner=false' - sql 'set exec_mem_limit=21G' - sql 'set be_number_for_test=3' - sql 'set enable_runtime_filter_prune=false' - sql 'set parallel_pipeline_task_num=8' - sql 'set forbid_unknown_col_stats=false' - sql 'set enable_stats=false' - sql "set runtime_filter_type=8" - sql 'set broadcast_row_count_limit = 30000000' - sql 'set enable_nereids_timeout = false' - sql 'SET enable_pipeline_engine = true' - sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - sql "set memo_max_group_expression_size = 1000000" - - def ds64 = ''' - with cs_ui as - (select cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), - cross_sales as - (select i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('blanched','medium','brown','chocolate','burlywood','drab') and - i_current_price between 23 and 23 + 10 and - i_current_price between 23 + 1 and 23 + 15 - group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year - ) - select cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt - from cross_sales cs1,cross_sales cs2 - where cs1.item_sk=cs2.item_sk and - cs1.syear = 2001 and - cs2.syear = 2001 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip - order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1; - - ''' - - qt_ds_shape_64 'explain shape plan ' + ds64 - -} diff --git a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query64.groovy b/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query64.groovy deleted file mode 100644 index ea7d7844ea939e..00000000000000 --- a/regression-test/suites/shape_check/tpcds_sf100/rf_prune/query64.groovy +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -suite("query64") { - String db = context.config.getDbNameByFile(new File(context.file.parent)) - if (isCloudMode()) { - return - } - sql "use ${db}" - sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=false' - sql 'set enable_fallback_to_original_planner=false' - sql 'set exec_mem_limit=21G' - sql 'set be_number_for_test=3' - sql 'set parallel_pipeline_task_num=8; ' - sql 'set forbid_unknown_col_stats=true' - sql 'set enable_nereids_timeout = false' - sql 'set runtime_filter_type=8' - sql 'set enable_runtime_filter_prune=true' - sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - sql "set memo_max_group_expression_size = 1000000" - - def ds = """with cs_ui as - (select cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -cross_sales as - (select i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('blanched','medium','brown','chocolate','burlywood','drab') and - i_current_price between 23 and 23 + 10 and - i_current_price between 23 + 1 and 23 + 15 -group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year -) -select cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt -from cross_sales cs1,cross_sales cs2 -where cs1.item_sk=cs2.item_sk and - cs1.syear = 2001 and - cs2.syear = 2001 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip -order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1""" - qt_ds_shape_64 """ - explain shape plan - ${ds} - """ -} diff --git a/regression-test/suites/shape_check/tpcds_sf100/shape/query64.groovy b/regression-test/suites/shape_check/tpcds_sf100/shape/query64.groovy deleted file mode 100644 index 557b8be4ba9b94..00000000000000 --- a/regression-test/suites/shape_check/tpcds_sf100/shape/query64.groovy +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -suite("query64") { - String db = context.config.getDbNameByFile(new File(context.file.parent)) - if (isCloudMode()) { - return - } - sql "use ${db}" - sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=false' - sql 'set enable_fallback_to_original_planner=false' - sql 'set exec_mem_limit=21G' - sql 'set be_number_for_test=3' - sql 'set parallel_pipeline_task_num=8; ' - sql 'set forbid_unknown_col_stats=true' - sql 'set enable_nereids_timeout = false' - sql 'set enable_runtime_filter_prune=false' - sql 'set runtime_filter_type=8' - sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - sql "set memo_max_group_expression_size = 1000000" - - def ds = """with cs_ui as - (select cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -cross_sales as - (select i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('blanched','medium','brown','chocolate','burlywood','drab') and - i_current_price between 23 and 23 + 10 and - i_current_price between 23 + 1 and 23 + 15 -group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year -) -select cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt -from cross_sales cs1,cross_sales cs2 -where cs1.item_sk=cs2.item_sk and - cs1.syear = 2001 and - cs2.syear = 2001 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip -order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1""" - qt_ds_shape_64 """ - explain shape plan - ${ds} - """ -} diff --git a/regression-test/suites/shape_check/tpcds_sf1000/hint/query64.groovy b/regression-test/suites/shape_check/tpcds_sf1000/hint/query64.groovy deleted file mode 100644 index c548c8f6ce815c..00000000000000 --- a/regression-test/suites/shape_check/tpcds_sf1000/hint/query64.groovy +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -suite("query64") { - String db = context.config.getDbNameByFile(new File(context.file.parent)) - if (isCloudMode()) { - return - } - sql "use ${db}" - sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=false' - sql 'set enable_fallback_to_original_planner=false' - sql 'set exec_mem_limit=21G' - sql 'set be_number_for_test=3' - sql 'set parallel_pipeline_task_num=8; ' - sql 'set forbid_unknown_col_stats=true' - sql 'set enable_nereids_timeout = false' - sql 'set enable_runtime_filter_prune=false' - sql 'set runtime_filter_type=8' - sql 'set dump_nereids_memo=false' - sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - - sql "set memo_max_group_expression_size = 1000000" - - def ds = """with cs_ui as - (select cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -cross_sales as - (select i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('orange','lace','lawn','misty','blush','pink') and - i_current_price between 48 and 48 + 10 and - i_current_price between 48 + 1 and 48 + 15 -group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year -) -select cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt -from cross_sales cs1,cross_sales cs2 -where cs1.item_sk=cs2.item_sk and - cs1.syear = 1999 and - cs2.syear = 1999 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip -order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1""" - qt_ds_shape_64 ''' - explain shape plan - with cs_ui as - (select - /*+ leading(catalog_sales shuffle catalog_returns) */ - cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -cross_sales as - (select - /*+ leading( {store_sales {{customer d2} cd2}} cd1 d3 item {hd1 ib1} store_returns ad1 hd2 ad2 ib2 d1 store promotion cs_ui) */ - i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('orange','lace','lawn','misty','blush','pink') and - i_current_price between 48 and 48 + 10 and - i_current_price between 48 + 1 and 48 + 15 -group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year -) -select -/*+ leading(cs1 shuffle cs2) */ - cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt -from cross_sales cs1,cross_sales cs2 -where cs1.item_sk=cs2.item_sk and - cs1.syear = 1999 and - cs2.syear = 1999 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip -order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1 - ''' -} diff --git a/regression-test/suites/shape_check/tpcds_sf1000/shape/query64.groovy b/regression-test/suites/shape_check/tpcds_sf1000/shape/query64.groovy deleted file mode 100644 index ef89e0bd0a7bc5..00000000000000 --- a/regression-test/suites/shape_check/tpcds_sf1000/shape/query64.groovy +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -suite("query64") { - String db = context.config.getDbNameByFile(new File(context.file.parent)) - if (isCloudMode()) { - return - } - sql "use ${db}" - sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=false' - sql 'set enable_fallback_to_original_planner=false' - sql 'set exec_mem_limit=21G' - sql 'set be_number_for_test=3' - sql 'set parallel_pipeline_task_num=8; ' - sql 'set forbid_unknown_col_stats=true' - sql 'set enable_nereids_timeout = false' - sql 'set enable_runtime_filter_prune=false' - sql 'set runtime_filter_type=8' - sql 'set dump_nereids_memo=false' - sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - sql "set memo_max_group_expression_size = 1000000" - - def ds = """with cs_ui as - (select cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -cross_sales as - (select i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('orange','lace','lawn','misty','blush','pink') and - i_current_price between 48 and 48 + 10 and - i_current_price between 48 + 1 and 48 + 15 -group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year -) -select cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt -from cross_sales cs1,cross_sales cs2 -where cs1.item_sk=cs2.item_sk and - cs1.syear = 1999 and - cs2.syear = 1999 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip -order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1""" -// qt_ds_shape_64 ''' -// explain shape plan -// with cs_ui as -// (select cs_item_sk -// ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund -// from catalog_sales -// ,catalog_returns -// where cs_item_sk = cr_item_sk -// and cs_order_number = cr_order_number -// group by cs_item_sk -// having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -// cross_sales as -// (select i_product_name product_name -// ,i_item_sk item_sk -// ,s_store_name store_name -// ,s_zip store_zip -// ,ad1.ca_street_number b_street_number -// ,ad1.ca_street_name b_street_name -// ,ad1.ca_city b_city -// ,ad1.ca_zip b_zip -// ,ad2.ca_street_number c_street_number -// ,ad2.ca_street_name c_street_name -// ,ad2.ca_city c_city -// ,ad2.ca_zip c_zip -// ,d1.d_year as syear -// ,d2.d_year as fsyear -// ,d3.d_year s2year -// ,count(*) cnt -// ,sum(ss_wholesale_cost) s1 -// ,sum(ss_list_price) s2 -// ,sum(ss_coupon_amt) s3 -// FROM store_sales -// ,store_returns -// ,cs_ui -// ,date_dim d1 -// ,date_dim d2 -// ,date_dim d3 -// ,store -// ,customer -// ,customer_demographics cd1 -// ,customer_demographics cd2 -// ,promotion -// ,household_demographics hd1 -// ,household_demographics hd2 -// ,customer_address ad1 -// ,customer_address ad2 -// ,income_band ib1 -// ,income_band ib2 -// ,item -// WHERE ss_store_sk = s_store_sk AND -// ss_sold_date_sk = d1.d_date_sk AND -// ss_customer_sk = c_customer_sk AND -// ss_cdemo_sk= cd1.cd_demo_sk AND -// ss_hdemo_sk = hd1.hd_demo_sk AND -// ss_addr_sk = ad1.ca_address_sk and -// ss_item_sk = i_item_sk and -// ss_item_sk = sr_item_sk and -// ss_ticket_number = sr_ticket_number and -// ss_item_sk = cs_ui.cs_item_sk and -// c_current_cdemo_sk = cd2.cd_demo_sk AND -// c_current_hdemo_sk = hd2.hd_demo_sk AND -// c_current_addr_sk = ad2.ca_address_sk and -// c_first_sales_date_sk = d2.d_date_sk and -// c_first_shipto_date_sk = d3.d_date_sk and -// ss_promo_sk = p_promo_sk and -// hd1.hd_income_band_sk = ib1.ib_income_band_sk and -// hd2.hd_income_band_sk = ib2.ib_income_band_sk and -// cd1.cd_marital_status <> cd2.cd_marital_status and -// i_color in ('orange','lace','lawn','misty','blush','pink') and -// i_current_price between 48 and 48 + 10 and -// i_current_price between 48 + 1 and 48 + 15 -// group by i_product_name -// ,i_item_sk -// ,s_store_name -// ,s_zip -// ,ad1.ca_street_number -// ,ad1.ca_street_name -// ,ad1.ca_city -// ,ad1.ca_zip -// ,ad2.ca_street_number -// ,ad2.ca_street_name -// ,ad2.ca_city -// ,ad2.ca_zip -// ,d1.d_year -// ,d2.d_year -// ,d3.d_year -// ) -// select cs1.product_name -// ,cs1.store_name -// ,cs1.store_zip -// ,cs1.b_street_number -// ,cs1.b_street_name -// ,cs1.b_city -// ,cs1.b_zip -// ,cs1.c_street_number -// ,cs1.c_street_name -// ,cs1.c_city -// ,cs1.c_zip -// ,cs1.syear -// ,cs1.cnt -// ,cs1.s1 as s11 -// ,cs1.s2 as s21 -// ,cs1.s3 as s31 -// ,cs2.s1 as s12 -// ,cs2.s2 as s22 -// ,cs2.s3 as s32 -// ,cs2.syear -// ,cs2.cnt -// from cross_sales cs1,cross_sales cs2 -// where cs1.item_sk=cs2.item_sk and -// cs1.syear = 1999 and -// cs2.syear = 1999 + 1 and -// cs2.cnt <= cs1.cnt and -// cs1.store_name = cs2.store_name and -// cs1.store_zip = cs2.store_zip -// order by cs1.product_name -// ,cs1.store_name -// ,cs2.cnt -// ,cs1.s1 -// ,cs2.s1 -// ''' -} diff --git a/regression-test/suites/shape_check/tpcds_sf1000_nopkfk/shape/query64.groovy b/regression-test/suites/shape_check/tpcds_sf1000_nopkfk/shape/query64.groovy deleted file mode 100644 index ef89e0bd0a7bc5..00000000000000 --- a/regression-test/suites/shape_check/tpcds_sf1000_nopkfk/shape/query64.groovy +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -suite("query64") { - String db = context.config.getDbNameByFile(new File(context.file.parent)) - if (isCloudMode()) { - return - } - sql "use ${db}" - sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=false' - sql 'set enable_fallback_to_original_planner=false' - sql 'set exec_mem_limit=21G' - sql 'set be_number_for_test=3' - sql 'set parallel_pipeline_task_num=8; ' - sql 'set forbid_unknown_col_stats=true' - sql 'set enable_nereids_timeout = false' - sql 'set enable_runtime_filter_prune=false' - sql 'set runtime_filter_type=8' - sql 'set dump_nereids_memo=false' - sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - sql "set memo_max_group_expression_size = 1000000" - - def ds = """with cs_ui as - (select cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -cross_sales as - (select i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('orange','lace','lawn','misty','blush','pink') and - i_current_price between 48 and 48 + 10 and - i_current_price between 48 + 1 and 48 + 15 -group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year -) -select cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt -from cross_sales cs1,cross_sales cs2 -where cs1.item_sk=cs2.item_sk and - cs1.syear = 1999 and - cs2.syear = 1999 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip -order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1""" -// qt_ds_shape_64 ''' -// explain shape plan -// with cs_ui as -// (select cs_item_sk -// ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund -// from catalog_sales -// ,catalog_returns -// where cs_item_sk = cr_item_sk -// and cs_order_number = cr_order_number -// group by cs_item_sk -// having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -// cross_sales as -// (select i_product_name product_name -// ,i_item_sk item_sk -// ,s_store_name store_name -// ,s_zip store_zip -// ,ad1.ca_street_number b_street_number -// ,ad1.ca_street_name b_street_name -// ,ad1.ca_city b_city -// ,ad1.ca_zip b_zip -// ,ad2.ca_street_number c_street_number -// ,ad2.ca_street_name c_street_name -// ,ad2.ca_city c_city -// ,ad2.ca_zip c_zip -// ,d1.d_year as syear -// ,d2.d_year as fsyear -// ,d3.d_year s2year -// ,count(*) cnt -// ,sum(ss_wholesale_cost) s1 -// ,sum(ss_list_price) s2 -// ,sum(ss_coupon_amt) s3 -// FROM store_sales -// ,store_returns -// ,cs_ui -// ,date_dim d1 -// ,date_dim d2 -// ,date_dim d3 -// ,store -// ,customer -// ,customer_demographics cd1 -// ,customer_demographics cd2 -// ,promotion -// ,household_demographics hd1 -// ,household_demographics hd2 -// ,customer_address ad1 -// ,customer_address ad2 -// ,income_band ib1 -// ,income_band ib2 -// ,item -// WHERE ss_store_sk = s_store_sk AND -// ss_sold_date_sk = d1.d_date_sk AND -// ss_customer_sk = c_customer_sk AND -// ss_cdemo_sk= cd1.cd_demo_sk AND -// ss_hdemo_sk = hd1.hd_demo_sk AND -// ss_addr_sk = ad1.ca_address_sk and -// ss_item_sk = i_item_sk and -// ss_item_sk = sr_item_sk and -// ss_ticket_number = sr_ticket_number and -// ss_item_sk = cs_ui.cs_item_sk and -// c_current_cdemo_sk = cd2.cd_demo_sk AND -// c_current_hdemo_sk = hd2.hd_demo_sk AND -// c_current_addr_sk = ad2.ca_address_sk and -// c_first_sales_date_sk = d2.d_date_sk and -// c_first_shipto_date_sk = d3.d_date_sk and -// ss_promo_sk = p_promo_sk and -// hd1.hd_income_band_sk = ib1.ib_income_band_sk and -// hd2.hd_income_band_sk = ib2.ib_income_band_sk and -// cd1.cd_marital_status <> cd2.cd_marital_status and -// i_color in ('orange','lace','lawn','misty','blush','pink') and -// i_current_price between 48 and 48 + 10 and -// i_current_price between 48 + 1 and 48 + 15 -// group by i_product_name -// ,i_item_sk -// ,s_store_name -// ,s_zip -// ,ad1.ca_street_number -// ,ad1.ca_street_name -// ,ad1.ca_city -// ,ad1.ca_zip -// ,ad2.ca_street_number -// ,ad2.ca_street_name -// ,ad2.ca_city -// ,ad2.ca_zip -// ,d1.d_year -// ,d2.d_year -// ,d3.d_year -// ) -// select cs1.product_name -// ,cs1.store_name -// ,cs1.store_zip -// ,cs1.b_street_number -// ,cs1.b_street_name -// ,cs1.b_city -// ,cs1.b_zip -// ,cs1.c_street_number -// ,cs1.c_street_name -// ,cs1.c_city -// ,cs1.c_zip -// ,cs1.syear -// ,cs1.cnt -// ,cs1.s1 as s11 -// ,cs1.s2 as s21 -// ,cs1.s3 as s31 -// ,cs2.s1 as s12 -// ,cs2.s2 as s22 -// ,cs2.s3 as s32 -// ,cs2.syear -// ,cs2.cnt -// from cross_sales cs1,cross_sales cs2 -// where cs1.item_sk=cs2.item_sk and -// cs1.syear = 1999 and -// cs2.syear = 1999 + 1 and -// cs2.cnt <= cs1.cnt and -// cs1.store_name = cs2.store_name and -// cs1.store_zip = cs2.store_zip -// order by cs1.product_name -// ,cs1.store_name -// ,cs2.cnt -// ,cs1.s1 -// ,cs2.s1 -// ''' -} diff --git a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query64.groovy b/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query64.groovy deleted file mode 100644 index 93f665fb540ade..00000000000000 --- a/regression-test/suites/shape_check/tpcds_sf10t_orc/shape/query64.groovy +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -suite("query64") { - String db = context.config.getDbNameByFile(new File(context.file.parent)) - if (isCloudMode()) { - return - } - sql """ - use ${db}; - set enable_nereids_planner=true; - set enable_nereids_distribute_planner=false; - set enable_fallback_to_original_planner=false; - set exec_mem_limit=21G; - set be_number_for_test=3; - set parallel_pipeline_task_num=8; - set forbid_unknown_col_stats=true; - set enable_nereids_timeout = false; - set enable_runtime_filter_prune=false; - set runtime_filter_type=8; - set dump_nereids_memo=false; - set disable_nereids_rules='PRUNE_EMPTY_PARTITION'; - set enable_fold_constant_by_be = false; - set push_topn_to_agg = true; - set TOPN_OPT_LIMIT_THRESHOLD = 1024; - set enable_parallel_result_sink=true; - set memo_max_group_expression_size = 1000000; - """ - qt_ds_shape_64 ''' - explain shape plan - with cs_ui as - (select cs_item_sk - ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund - from catalog_sales - ,catalog_returns - where cs_item_sk = cr_item_sk - and cs_order_number = cr_order_number - group by cs_item_sk - having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -cross_sales as - (select i_product_name product_name - ,i_item_sk item_sk - ,s_store_name store_name - ,s_zip store_zip - ,ad1.ca_street_number b_street_number - ,ad1.ca_street_name b_street_name - ,ad1.ca_city b_city - ,ad1.ca_zip b_zip - ,ad2.ca_street_number c_street_number - ,ad2.ca_street_name c_street_name - ,ad2.ca_city c_city - ,ad2.ca_zip c_zip - ,d1.d_year as syear - ,d2.d_year as fsyear - ,d3.d_year s2year - ,count(*) cnt - ,sum(ss_wholesale_cost) s1 - ,sum(ss_list_price) s2 - ,sum(ss_coupon_amt) s3 - FROM store_sales - ,store_returns - ,cs_ui - ,date_dim d1 - ,date_dim d2 - ,date_dim d3 - ,store - ,customer - ,customer_demographics cd1 - ,customer_demographics cd2 - ,promotion - ,household_demographics hd1 - ,household_demographics hd2 - ,customer_address ad1 - ,customer_address ad2 - ,income_band ib1 - ,income_band ib2 - ,item - WHERE ss_store_sk = s_store_sk AND - ss_sold_date_sk = d1.d_date_sk AND - ss_customer_sk = c_customer_sk AND - ss_cdemo_sk= cd1.cd_demo_sk AND - ss_hdemo_sk = hd1.hd_demo_sk AND - ss_addr_sk = ad1.ca_address_sk and - ss_item_sk = i_item_sk and - ss_item_sk = sr_item_sk and - ss_ticket_number = sr_ticket_number and - ss_item_sk = cs_ui.cs_item_sk and - c_current_cdemo_sk = cd2.cd_demo_sk AND - c_current_hdemo_sk = hd2.hd_demo_sk AND - c_current_addr_sk = ad2.ca_address_sk and - c_first_sales_date_sk = d2.d_date_sk and - c_first_shipto_date_sk = d3.d_date_sk and - ss_promo_sk = p_promo_sk and - hd1.hd_income_band_sk = ib1.ib_income_band_sk and - hd2.hd_income_band_sk = ib2.ib_income_band_sk and - cd1.cd_marital_status <> cd2.cd_marital_status and - i_color in ('azure','gainsboro','misty','blush','hot','lemon') and - i_current_price between 80 and 80 + 10 and - i_current_price between 80 + 1 and 80 + 15 -group by i_product_name - ,i_item_sk - ,s_store_name - ,s_zip - ,ad1.ca_street_number - ,ad1.ca_street_name - ,ad1.ca_city - ,ad1.ca_zip - ,ad2.ca_street_number - ,ad2.ca_street_name - ,ad2.ca_city - ,ad2.ca_zip - ,d1.d_year - ,d2.d_year - ,d3.d_year -) -select cs1.product_name - ,cs1.store_name - ,cs1.store_zip - ,cs1.b_street_number - ,cs1.b_street_name - ,cs1.b_city - ,cs1.b_zip - ,cs1.c_street_number - ,cs1.c_street_name - ,cs1.c_city - ,cs1.c_zip - ,cs1.syear - ,cs1.cnt - ,cs1.s1 as s11 - ,cs1.s2 as s21 - ,cs1.s3 as s31 - ,cs2.s1 as s12 - ,cs2.s2 as s22 - ,cs2.s3 as s32 - ,cs2.syear - ,cs2.cnt -from cross_sales cs1,cross_sales cs2 -where cs1.item_sk=cs2.item_sk and - cs1.syear = 1999 and - cs2.syear = 1999 + 1 and - cs2.cnt <= cs1.cnt and - cs1.store_name = cs2.store_name and - cs1.store_zip = cs2.store_zip -order by cs1.product_name - ,cs1.store_name - ,cs2.cnt - ,cs1.s1 - ,cs2.s1 - ''' -} diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy index 8f5297bfa29506..97f2c59ee8c4f8 100644 --- a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy +++ b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy @@ -50,6 +50,22 @@ suite("test_show_create_table_and_views_nereids", "show") { String rollupName = "${suiteName}_rollup" String likeName = "${suiteName}_like" + def forceReplicaAllocation = getFeConfig('force_olap_table_replication_allocation').trim() + def effectiveReplicaNum = 1 + if (!forceReplicaAllocation.isEmpty()) { + def matcher = forceReplicaAllocation =~ /^tag\.location\.default:\s*(1|3)$/ + assertTrue(matcher.matches(), + "Unsupported force_olap_table_replication_allocation: ${forceReplicaAllocation}") + effectiveReplicaNum = matcher.group(1).toInteger() + } else { + def forceReplicaNum = getFeConfig('force_olap_table_replication_num').toInteger() + if (forceReplicaNum > 0) { + assertTrue(forceReplicaNum in [1, 3], + "Unsupported force_olap_table_replication_num: ${forceReplicaNum}") + effectiveReplicaNum = forceReplicaNum + } + } + sql "SET enable_nereids_planner=true;" sql "SET enable_fallback_to_original_planner=false;" @@ -99,7 +115,7 @@ suite("test_show_create_table_and_views_nereids", "show") { (2, 200, 1111), (23, 900, 1)""" - qt_show "SHOW CREATE TABLE ${dbName}.${tableName}" + quickRunTest("show_initial_replica_${effectiveReplicaNum}", "SHOW CREATE TABLE ${dbName}.${tableName}") qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY user_id, good_id" sql "drop view if exists ${dbName}.${viewName};" @@ -132,15 +148,16 @@ suite("test_show_create_table_and_views_nereids", "show") { } qt_select "SELECT user_id, SUM(cost) FROM ${dbName}.${tableName} GROUP BY user_id ORDER BY user_id" - qt_show "SHOW CREATE TABLE ${dbName}.${tableName}" + quickRunTest("show_after_rollup_replica_${effectiveReplicaNum}", "SHOW CREATE TABLE ${dbName}.${tableName}") // create like sql "CREATE TABLE ${dbName}.${likeName} LIKE ${dbName}.${tableName}" - qt_show "SHOW CREATE TABLE ${dbName}.${likeName}" + quickRunTest("show_like_replica_${effectiveReplicaNum}", "SHOW CREATE TABLE ${dbName}.${likeName}") // create like with rollup sql "CREATE TABLE ${dbName}.${likeName}_with_rollup LIKE ${dbName}.${tableName} WITH ROLLUP" - qt_show "SHOW CREATE TABLE ${dbName}.${likeName}_with_rollup" + quickRunTest("show_like_with_rollup_replica_${effectiveReplicaNum}", + "SHOW CREATE TABLE ${dbName}.${likeName}_with_rollup") sql "DROP TABLE IF EXISTS ${dbName}.${likeName}_with_rollup FORCE" sql "DROP TABLE ${dbName}.${likeName} FORCE" @@ -148,4 +165,3 @@ suite("test_show_create_table_and_views_nereids", "show") { sql "DROP TABLE ${dbName}.${tableName} FORCE" sql "DROP DATABASE ${dbName} FORCE" } - diff --git a/regression-test/suites/table_stream_p0/test_min_delta_stream.groovy b/regression-test/suites/table_stream_p0/test_min_delta_stream.groovy index 634367f14d4acd..bd85855cd7939d 100644 --- a/regression-test/suites/table_stream_p0/test_min_delta_stream.groovy +++ b/regression-test/suites/table_stream_p0/test_min_delta_stream.groovy @@ -473,7 +473,7 @@ suite("test_min_delta_stream", "nonConcurrent") { """ assertEquals(1, ukDeleteBeforeRows.size()) assertEquals("10", ukDeleteBeforeRows[0][0].toString()) - assertEquals("101", ukDeleteBeforeRows[0][1].toString()) + assertEquals("100", ukDeleteBeforeRows[0][1].toString()) assertEquals("DELETE", ukDeleteBeforeRows[0][2].toString()) // 9) show_initial_rows=true: diff --git a/regression-test/suites/table_stream_p0/test_mow_min_delta_delete_before.groovy b/regression-test/suites/table_stream_p0/test_mow_min_delta_delete_before.groovy new file mode 100644 index 00000000000000..8b9633d3a2123b --- /dev/null +++ b/regression-test/suites/table_stream_p0/test_mow_min_delta_delete_before.groovy @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_mow_min_delta_delete_before", "nonConcurrent") { + if (isCloudMode()) { + return + } + sql "DROP DATABASE IF EXISTS test_mow_min_delta_delete_before_db" + sql "CREATE DATABASE test_mow_min_delta_delete_before_db" + sql "USE test_mow_min_delta_delete_before_db" + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + + try { + sql "DROP STREAM IF EXISTS mow_min_delta_bug_stream" + sql "DROP TABLE IF EXISTS mow_min_delta_bug" + + // MoW UNIQUE KEY + MIN_DELTA: when an UPDATE and a DELETE on the same key + // are folded into a single DELETE, the emitted DELETE row must carry the + // pre-delete snapshot (the UPDATE_BEFORE old value), not the UPDATE_AFTER + // new value nor the DELETE tombstone value. + sql """ + CREATE TABLE mow_min_delta_bug ( + id BIGINT, + v1 INT + ) ENGINE=OLAP + UNIQUE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "binlog.enable" = "true", + "binlog.format" = "ROW", + "binlog.need_historical_value" = "true" + ) + """ + // Seed old value before the stream starts so the in-window change is an + // UPDATE (not a fresh APPEND). + sql "INSERT INTO mow_min_delta_bug VALUES (1, 10)" + sql """ + CREATE STREAM mow_min_delta_bug_stream + ON TABLE mow_min_delta_bug + PROPERTIES ( + "type" = "min_delta", + "show_initial_rows" = "false" + ) + """ + sql "sync" + sql "INSERT INTO mow_min_delta_bug VALUES (1, 11)" + sql "DELETE FROM mow_min_delta_bug WHERE id = 1" + sql "sync" + sleep(1200) + + // Expect a single DELETE row keeping the old value 10. + order_qt_mow_min_delta_delete_before """ + SELECT id, v1, __DORIS_STREAM_CHANGE_TYPE_COL__ + FROM mow_min_delta_bug_stream + ORDER BY id, v1, __DORIS_STREAM_CHANGE_TYPE_COL__ + """ + + sql "DROP STREAM IF EXISTS mow_min_delta_del_ins_del_stream" + sql "DROP TABLE IF EXISTS mow_min_delta_del_ins_del" + + // MoW UNIQUE KEY + MIN_DELTA: when a DELETE, a re-INSERT and another DELETE + // on the same key are folded into a single DELETE, the first op is a DELETE, + // which means the key already existed before the window. The emitted DELETE + // row must carry the first op's pre-delete snapshot (the original value 20), + // not the re-inserted value 21. + sql """ + CREATE TABLE mow_min_delta_del_ins_del ( + id BIGINT, + v1 INT + ) ENGINE=OLAP + UNIQUE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "binlog.enable" = "true", + "binlog.format" = "ROW", + "binlog.need_historical_value" = "true" + ) + """ + // Seed old value before the stream starts so the first in-window DELETE + // refers to a pre-existing row. + sql "INSERT INTO mow_min_delta_del_ins_del VALUES (2, 20)" + sql """ + CREATE STREAM mow_min_delta_del_ins_del_stream + ON TABLE mow_min_delta_del_ins_del + PROPERTIES ( + "type" = "min_delta", + "show_initial_rows" = "false" + ) + """ + sql "sync" + sql "DELETE FROM mow_min_delta_del_ins_del WHERE id = 2" + sql "INSERT INTO mow_min_delta_del_ins_del VALUES (2, 21)" + sql "DELETE FROM mow_min_delta_del_ins_del WHERE id = 2" + sql "sync" + sleep(1200) + + // Expect a single DELETE row keeping the old value 20. + order_qt_mow_min_delta_del_ins_del """ + SELECT id, v1, __DORIS_STREAM_CHANGE_TYPE_COL__ + FROM mow_min_delta_del_ins_del_stream + ORDER BY id, v1, __DORIS_STREAM_CHANGE_TYPE_COL__ + """ + } finally { + sql "DROP DATABASE IF EXISTS test_mow_min_delta_delete_before_db" + } +} diff --git a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.groovy b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.groovy index fb4f1b10a17cfb..5054c45b99217b 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.groovy @@ -36,6 +36,7 @@ suite("test_auto_inc_replica_consistency") { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", diff --git a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.groovy b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.groovy index 0c62427f94bd41..d83bbd42ca09c8 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.groovy @@ -36,6 +36,7 @@ suite("test_f_seq_publish_read_from_old") { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", diff --git a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.groovy b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.groovy index 7f257a2c336a1b..0f29ad9065424c 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.groovy @@ -36,6 +36,7 @@ suite("test_flexible_partial_update_publish_conflict_seq") { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_f_seq_read_from_old.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_f_seq_read_from_old.groovy index 4eed4ada2bd48a..d1cafa5f493e16 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/test_f_seq_read_from_old.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/test_f_seq_read_from_old.groovy @@ -39,6 +39,7 @@ suite('test_f_seq_read_from_old') { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", @@ -85,4 +86,4 @@ suite('test_f_seq_read_from_old') { qt_sql3 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} order by k,__DORIS_VERSION_COL__;" } -} \ No newline at end of file +} diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.groovy index 3fad13d5304985..b4c26493b2c0fe 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.groovy @@ -43,6 +43,7 @@ suite('test_flexible_partial_update_delete_sign') { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", @@ -93,6 +94,7 @@ suite('test_flexible_partial_update_delete_sign') { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", @@ -130,6 +132,7 @@ suite('test_flexible_partial_update_delete_sign') { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", @@ -161,6 +164,7 @@ suite('test_flexible_partial_update_delete_sign') { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", @@ -193,6 +197,7 @@ suite('test_flexible_partial_update_delete_sign') { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", @@ -264,6 +269,7 @@ suite('test_flexible_partial_update_delete_sign') { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", @@ -336,6 +342,7 @@ suite('test_flexible_partial_update_delete_sign') { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", @@ -353,4 +360,4 @@ suite('test_flexible_partial_update_delete_sign') { qt_insert_after_delete_3_4 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" } -} \ No newline at end of file +} diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.groovy index 8296531ea9f68c..6b865b01f1fab7 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.groovy @@ -287,6 +287,7 @@ suite('test_flexible_partial_update_seq_col') { DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES ( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "enable_unique_key_skip_bitmap_column" = "true", "function_column.sequence_col" = "c1"); @@ -321,6 +322,7 @@ suite('test_flexible_partial_update_seq_col') { DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES ( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "enable_unique_key_skip_bitmap_column" = "true", "function_column.sequence_col" = "c1"); @@ -343,4 +345,4 @@ suite('test_flexible_partial_update_seq_col') { sql "set skip_delete_bitmap=false;" sql "sync;" } -} \ No newline at end of file +} diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.groovy index f1a6ea813e85da..bd07a6cf5f454f 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.groovy @@ -35,6 +35,7 @@ suite("test_p_seq_publish_read_from_old") { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "function_column.sequence_col" = "v1", diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.groovy index 643ded4ef09685..73abf34c1e2099 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.groovy @@ -40,6 +40,7 @@ suite('test_partial_update_seq_read_from_old') { ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "enable_unique_key_skip_bitmap_column" = "true", @@ -65,4 +66,4 @@ suite('test_partial_update_seq_read_from_old') { qt_sql3 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__;" } -} \ No newline at end of file +} diff --git a/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy b/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy new file mode 100644 index 00000000000000..f04bda1fdd32e0 --- /dev/null +++ b/regression-test/suites/variant_p0/test_topn_lazy_materialize_sparse_variant.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_topn_lazy_materialize_sparse_variant", "p0") { + sql "set default_variant_enable_doc_mode = false" + sql "set default_variant_max_subcolumns_count = 1" + sql "set default_variant_sparse_hash_shard_count = 2" + sql "set use_v3_storage_format = true" + sql "set enable_file_cache = true" + sql "set enable_segment_limit_pushdown = false" + sql "set topn_lazy_materialization_threshold = 1024" + + sql "DROP TABLE IF EXISTS test_topn_lazy_materialize_sparse_variant" + sql """ + CREATE TABLE test_topn_lazy_materialize_sparse_variant ( + pk INT, + col_bigint BIGINT NULL, + col_json JSON NOT NULL, + col_variant VARIANT NULL + ) ENGINE = OLAP + DUPLICATE KEY(pk) + PARTITION BY RANGE(pk) ( + PARTITION p0 VALUES LESS THAN ('4'), + PARTITION p1 VALUES LESS THAN ('64'), + PARTITION p2 VALUES LESS THAN ('256'), + PARTITION pmax VALUES LESS THAN ('2147483647') + ) + DISTRIBUTED BY HASH(pk) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "storage_format" = "V3" + ) + """ + + sql """ + INSERT INTO test_topn_lazy_materialize_sparse_variant VALUES + (19, -9223372036854775808, '{"word":"hot","n":7}', '[1,2,3]'), + (18, -9223372036854775808, '{"word":"hot","n":7}', '{"word":"hot","n":7}'), + (17, -1, '{"k":1}', '{}'), + (16, -906638365906932436, '{"word":"hot","n":7}', '{"k":1}'), + (15, -1, '[1,2,3]', '{}'), + (14, -9223372036854775808, '{"word":"hot","n":7}', '[1,2,3]'), + (13, 5777142737451291289, '{}', '{"word":"hot","n":7}'), + (12, 1, '{"word":"hot","n":7}', NULL), + (11, 42, '[1,2,3]', NULL), + (10, 1, '{}', '{"word":"hot","n":7}'), + (9, NULL, '{}', '{}'), + (8, NULL, '{"k":1}', '{"k":1}'), + (7, 0, '[1,2,3]', '[1,2,3]'), + (6, 1, '[1,2,3]', '{}'), + (5, -3610716764638269764, '{"k":1}', '[1,2,3]'), + (4, -9223372036854775808, '[1,2,3]', '{"k":1}'), + (3, 42, '{"k":1}', '{}'), + (2, -9223372036854775808, '{}', '{}'), + (1, -1, '{}', '[1,2,3]'), + (0, 1, '{}', '{}') + """ + + def topnQuery = """ + SELECT pk, col_bigint, col_json, col_variant + FROM test_topn_lazy_materialize_sparse_variant + WHERE ABS(pk % 3) IN (0, 1, 3) + ORDER BY pk ASC + LIMIT 128 + """ + explain { + sql topnQuery + contains("MaterializeNode") + } + qt_topn_lazy_sparse_variant topnQuery +} diff --git a/run-regression-test.sh b/run-regression-test.sh index 87143a2cda5545..2630bfb9a36bb1 100755 --- a/run-regression-test.sh +++ b/run-regression-test.sh @@ -23,6 +23,63 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" DORIS_HOME="${ROOT}" +java_major_version() { + local java_cmd="$1" + local spec_version + + spec_version="$("${java_cmd}" -XshowSettings:properties -version 2>&1 \ + | awk -F'= ' '/java.specification.version =/ {print $2; exit}')" + # Cygwin's Java properties output uses CRLF line endings. + spec_version="${spec_version//$'\r'/}" + if [[ "${spec_version}" == 1.* ]]; then + spec_version="${spec_version#1.}" + fi + echo "${spec_version%%.*}" +} + +is_jdk17_home() { + local java_home="$1" + [[ -n "${java_home}" ]] && [[ -x "${java_home}/bin/java" ]] \ + && [[ -x "${java_home}/bin/javac" ]] \ + && [[ "$(java_major_version "${java_home}/bin/java")" == "17" ]] +} + +find_jdk17_home() { + local candidate + + for candidate in "${JAVA_HOME:-}" "${JDK_17:-}"; do + if is_jdk17_home "${candidate}"; then + echo "${candidate}" + return 0 + fi + done + + if [[ "$(uname -s)" == "Darwin" ]] && [[ -x /usr/libexec/java_home ]]; then + candidate="$(/usr/libexec/java_home -v 17 2>/dev/null || true)" + if is_jdk17_home "${candidate}"; then + echo "${candidate}" + return 0 + fi + elif [[ -d /usr/lib/jvm ]]; then + while IFS= read -r candidate; do + if is_jdk17_home "${candidate}"; then + echo "${candidate}" + return 0 + fi + done < <(find /usr/lib/jvm -mindepth 1 -maxdepth 1 \( -type d -o -type l \) 2>/dev/null | sort) + fi + + echo "Error: JDK 17 is required. Set JAVA_HOME or JDK_17, or install JDK 17." >&2 + return 1 +} + +JAVA_HOME="$(find_jdk17_home)" +export JAVA_HOME +export PATH="${JAVA_HOME}/bin:${PATH}" +export JAVA="${JAVA_HOME}/bin/java" +JAVA_MAJOR_VERSION="$(java_major_version "${JAVA}")" +echo "Using JDK ${JAVA_MAJOR_VERSION}: ${JAVA_HOME}" + # Check args usage() { echo " @@ -212,7 +269,8 @@ if ! test -f ${RUN_JAR:+${RUN_JAR}}; then # Then package with retry echo "Building package..." - execute_maven_with_retry "${MVN_CMD} clean package -B -DskipTests=true -Dmaven.javadoc.skip=true" || { + # Keep framework unit tests in the standard compile path so pipeline builds enforce framework regressions. + execute_maven_with_retry "${MVN_CMD} clean package -B -Dmaven.javadoc.skip=true" || { echo "Failed to build package" exit 1 } @@ -226,30 +284,8 @@ if ! test -f ${RUN_JAR:+${RUN_JAR}}; then mkdir -p "${DORIS_HOME}"/regression-test/suites/javaudf_p0/jars cd "${DORIS_HOME}"/regression-test/java-udf-src || { echo "Failed to change directory to java-udf-src"; exit 1; } - # The Java UDF jar must be built with JDK 8 (source/target 8, loaded by the BE embedded - # JVM), while the framework above is built with the ambient JDK (>=17 in CI). Switch to a - # JDK 8 just for this module, falling back to the ambient JDK if none is installed - # (e.g. local dev boxes). - udf_prev_java_home="${JAVA_HOME:-}" - udf_jdk8_home="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' 2>/dev/null | sed -n '1p')" - if [[ -n "${udf_jdk8_home}" ]]; then - echo "Building Java UDF with JDK 8 at ${udf_jdk8_home}" - export JAVA_HOME="${udf_jdk8_home}" - else - echo "WARNING: no JDK 8 found under /usr/lib/jvm; building Java UDF with ${JAVA_HOME:-the current JDK}" - fi - - # Build UDF with retry - udf_build_rc=0 - execute_maven_with_retry "${MVN_CMD} clean package -B -DskipTests=true -Dmaven.javadoc.skip=true" || udf_build_rc=1 - - # Restore the framework JDK regardless of the UDF build result. - if [[ -n "${udf_prev_java_home}" ]]; then - export JAVA_HOME="${udf_prev_java_home}" - else - unset JAVA_HOME - fi - if [[ "${udf_build_rc}" -ne 0 ]]; then + # Build the UDF test jar with the same JDK 17 used by the regression framework. + if ! execute_maven_with_retry "${MVN_CMD} clean package -B -DskipTests=true -Dmaven.javadoc.skip=true"; then echo "Failed to build UDF package" exit 1 fi @@ -263,22 +299,6 @@ if ! test -f ${RUN_JAR:+${RUN_JAR}}; then cd "${DORIS_HOME}" || { echo "Failed to return to DORIS_HOME"; exit 1; } fi -# check java home -if [[ -z "${JAVA_HOME}" ]]; then - echo "Error: JAVA_HOME is not set" - exit 1 -fi - -# check java version -export JAVA="${JAVA_HOME}/bin/java" -JAVA_SPEC_VERSION="$("${JAVA}" -XshowSettings:properties -version 2>&1 \ - | awk -F'= ' '/java.specification.version =/ {print $2; exit}')" -JAVA_MAJOR_VERSION="${JAVA_SPEC_VERSION%%.*}" -if [[ "${JAVA_SPEC_VERSION}" == 1.* ]]; then - JAVA_MAJOR_VERSION="${JAVA_SPEC_VERSION#1.}" - JAVA_MAJOR_VERSION="${JAVA_MAJOR_VERSION%%.*}" -fi - # Arrow Flight SQL JDBC needs java.nio opened when the regression framework runs on JDK 17+. if [[ -n "${JAVA_MAJOR_VERSION}" ]] && [[ "${JAVA_MAJOR_VERSION}" -ge 17 ]] \ && [[ " ${JAVA_OPTS:-} " != *"--add-opens=java.base/java.nio="* ]]; then diff --git a/thirdparty/LICENSE.txt b/thirdparty/LICENSE.txt index 7db23714ed12d1..1c29d4f1a8a4e3 100644 --- a/thirdparty/LICENSE.txt +++ b/thirdparty/LICENSE.txt @@ -43,6 +43,16 @@ Apache 2.0 License (found in the LICENSE.Apache file in the root directory) Doris selects Apache 2.0 License of RocksDB. +============================================================================================ +Lance-C +source: https://github.com/lance-format/lance-c + +Lance-C is licensed under Apache License Version 2.0. + +The static library is built from Rust crates. The transitive Rust crate license +list for normal/build dependencies is maintained in +`dist/licenses/LICENSE-lance-c-rust-crates.txt`. + ============================================================================================ LZO source: https://github.com/nemequ/lzo diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 180333d150fbe4..2a468540484fd3 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -2126,6 +2126,68 @@ build_paimon_cpp() { echo "Paimon-cpp internal dependencies installed successfully" } +# lance-c +build_lance_c() { + check_if_source_exist "${LANCE_C_SOURCE}" + cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" + + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + + local cargo_bin="${LANCE_C_CARGO:-${CARGO:-cargo}}" + if ! command -v "${cargo_bin}" >/dev/null 2>&1; then + echo "cargo is required to build lance-c. Install Rust 1.91.0 or set LANCE_C_CARGO." + exit 1 + fi + if [[ ! -x "${TP_INSTALL_DIR}/bin/protoc" ]]; then + echo "protoc is required to build lance-c. Build protobuf first." + exit 1 + fi + + local required_rust_version="1.91.0" + local cargo_env=( + "CARGO_BUILD_JOBS=${PARALLEL}" + "CARGO_TARGET_DIR=${PWD}/${BUILD_DIR}" + "PROTOC=${TP_INSTALL_DIR}/bin/protoc" + ) + if command -v rustup >/dev/null 2>&1 && [[ -z "${RUSTUP_TOOLCHAIN}" ]]; then + if ! rustup toolchain list | grep -Eq '^1\.91\.0([[:space:]-]|$)'; then + rustup toolchain install "${required_rust_version}" --profile minimal + fi + cargo_env+=("RUSTUP_TOOLCHAIN=${required_rust_version}") + fi + + local cargo_version + if ! cargo_version="$(env "${cargo_env[@]}" "${cargo_bin}" --version | awk '{print $2}')"; then + echo "failed to get cargo version for lance-c. Install Rust ${required_rust_version} or set LANCE_C_CARGO/RUSTUP_TOOLCHAIN." + exit 1 + fi + if [[ "${cargo_version}" != "${required_rust_version}" ]]; then + echo "lance-c requires Rust/Cargo ${required_rust_version}, but found ${cargo_version}." + echo "Install Rust ${required_rust_version} or set LANCE_C_CARGO/RUSTUP_TOOLCHAIN." + exit 1 + fi + + if [[ "${KERNEL}" != 'Darwin' ]]; then + cargo_env+=("CFLAGS=${CFLAGS:-} -std=gnu17") + fi + + local cargo_args=(build --release --locked) + if [[ "$(echo "${LANCE_C_CARGO_OFFLINE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then + cargo_args+=(--offline) + fi + env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}" + + mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64" + rm -rf "${TP_INSTALL_DIR}/include/lance" + cp -av include/lance "${TP_INSTALL_DIR}/include/" + cp -v "${BUILD_DIR}/release/liblance_c.a" "${TP_INSTALL_DIR}/lib64/" + + if [[ "${STRIP_TP_LIB}" = "ON" && "${KERNEL}" != 'Darwin' ]]; then + strip --strip-debug --strip-unneeded "${TP_INSTALL_DIR}/lib64/liblance_c.a" + fi +} + if [[ "${#packages[@]}" -eq 0 ]]; then packages=( jindofs @@ -2166,6 +2228,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then cares grpc # after cares, protobuf arrow + lance_c s2 bitshuffle croaringbitmap @@ -2300,6 +2363,7 @@ cleanup_package_source() { juicefs) src_var="JUICEFS_SOURCE" ;; pugixml) src_var="PUGIXML_SOURCE" ;; paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;; + lance_c) src_var="LANCE_C_SOURCE" ;; aws_sdk) src_var="AWS_SDK_SOURCE" ;; lzma) src_var="LZMA_SOURCE" ;; xml2) src_var="XML2_SOURCE" ;; diff --git a/thirdparty/patches/brpc-1.4.0-secondary-package-name.patch b/thirdparty/patches/brpc-1.4.0-secondary-package-name.patch index 8d8a37ec0e71a0..34a519314f2990 100644 --- a/thirdparty/patches/brpc-1.4.0-secondary-package-name.patch +++ b/thirdparty/patches/brpc-1.4.0-secondary-package-name.patch @@ -17,7 +17,7 @@ index 2087cbcf..7aede561 100644 Tabbed* tabbed = dynamic_cast(service); for (int i = 0; i < sd->method_count(); ++i) { const google::protobuf::MethodDescriptor* md = sd->method(i); -@@ -1282,6 +1290,14 @@ int Server::AddServiceInternal(google::protobuf::Service* service, +@@ -1282,6 +1290,16 @@ int Server::AddServiceInternal(google::protobuf::Service* service, mp.method = md; mp.status = new MethodStatus; _method_map[md->full_name()] = mp; @@ -27,17 +27,21 @@ index 2087cbcf..7aede561 100644 + secondary_method_name.append(secondary_full_name); + secondary_method_name.push_back('.'); + secondary_method_name.append(md->name()); -+ _method_map[secondary_method_name] = mp; ++ MethodProperty secondary_mp = mp; ++ secondary_mp.own_method_status = false; ++ _method_map[secondary_method_name] = secondary_mp; + } if (is_idl_support && sd->name() != sd->full_name()/*has ns*/) { MethodProperty mp2 = mp; mp2.own_method_status = false; -@@ -1306,6 +1322,9 @@ int Server::AddServiceInternal(google::protobuf::Service* service, +@@ -1306,6 +1324,11 @@ int Server::AddServiceInternal(google::protobuf::Service* service, is_builtin_service, svc_opt.ownership, service, NULL }; _fullname_service_map[sd->full_name()] = ss; _service_map[sd->name()] = ss; + if (!secondary_full_name.empty()) { -+ _fullname_service_map[secondary_full_name] = ss; ++ ServiceProperty secondary_ss = ss; ++ secondary_ss.ownership = SERVER_DOESNT_OWN_SERVICE; ++ _fullname_service_map[secondary_full_name] = secondary_ss; + } if (is_builtin_service) { ++_builtin_service_count; diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index af46e566b8a30f..f8d18a3a2bea16 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -573,6 +573,12 @@ PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" PAIMON_CPP_SOURCE="doris-thirdparty-paimon-cpp-0a4f4e2" PAIMON_CPP_MD5SUM="b8599a0421dbf1ec05e2f1a481d64e87" +# lance-c +LANCE_C_DOWNLOAD="https://github.com/lance-format/lance-c/archive/refs/tags/v0.1.2.tar.gz" +LANCE_C_NAME="lance-c-v0.1.2.tar.gz" +LANCE_C_SOURCE="lance-c-0.1.2" +LANCE_C_MD5SUM="eb6ec9bc63fa5245864282f24b521d0b" + # all thirdparties which need to be downloaded is set in array TP_ARCHIVES export TP_ARCHIVES=( 'LIBEVENT' @@ -658,6 +664,7 @@ export TP_ARCHIVES=( 'JUICEFS' 'PUGIXML' 'PAIMON_CPP' + 'LANCE_C' ) if [[ "$(uname -s)" == 'Darwin' ]]; then diff --git a/tools/cost_model_evaluate/requirements.txt b/tools/cost_model_evaluate/requirements.txt index 05497a9893e5ff..b91cc1c74cea76 100644 --- a/tools/cost_model_evaluate/requirements.txt +++ b/tools/cost_model_evaluate/requirements.txt @@ -16,4 +16,4 @@ # under the License. matplotlib==3.7.0 -mysql_connector_repackaged==0.3.1 +mysql-connector-python>=8.0.33,<9