From d863d3b37a71b20577df0d5bb6e231896804ba58 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:05:27 +0800 Subject: [PATCH 1/6] fix(release): make source archives reproducible (#177) (#178) --- .github/workflows/release_candidate.yaml | 4 + scripts/releasing/README.md | 53 ++-- scripts/releasing/create_source_release.sh | 40 ++- scripts/releasing/release_rc.sh | 126 ++++++-- scripts/releasing/tests/test_release_tools.py | 282 ++++++++++++++++++ scripts/releasing/verify_release_candidate.sh | 45 ++- 6 files changed, 495 insertions(+), 55 deletions(-) diff --git a/.github/workflows/release_candidate.yaml b/.github/workflows/release_candidate.yaml index 28a4b093..859b2578 100644 --- a/.github/workflows/release_candidate.yaml +++ b/.github/workflows/release_candidate.yaml @@ -78,9 +78,13 @@ jobs: --output-dir release/ci - name: Audit source archive + env: + # The creator must ignore caller-provided gzip defaults. + GZIP: "-9" run: | scripts/releasing/verify_release_candidate.sh \ --allow-unsigned \ + --git-ref HEAD \ --skip-build \ "release/ci/apache-paimon-cpp-${RELEASE_VERSION}-src.tgz" diff --git a/scripts/releasing/README.md b/scripts/releasing/README.md index ee558c1e..a251bed8 100644 --- a/scripts/releasing/README.md +++ b/scripts/releasing/README.md @@ -34,9 +34,9 @@ Before starting a release: - obtain an ASF code-signing key, publish it through the ASF account system, and make sure it is present in [Paimon KEYS](https://downloads.apache.org/paimon/KEYS); -- install `git`, `gpg`, `svn`, `gh`, `python3`, `curl` or `wget`, Java, CMake, - Ninja, and the toolchain needed by `ci/scripts/build_paimon.sh` (Java is - required by Apache RAT); +- install `git`, GNU `gzip`, `gpg`, `svn`, `gh`, `python3`, `curl` or `wget`, + Java, CMake, Ninja, and the toolchain needed by + `ci/scripts/build_paimon.sh` (Java is required by Apache RAT); - authenticate `gh` with access to read GitHub Actions runs in `apache/paimon-cpp`; - make sure the Apache Git remote points directly to @@ -44,6 +44,15 @@ Before starting a release: - prepare and merge a release-preparation PR that updates the release notes and all version metadata, and passes the normal and release-candidate workflows. +The source archive uses GNU gzip with fixed options so that macOS and Linux +produce the same bytes. On macOS, install Homebrew gzip and either put it first +on `PATH` or select it explicitly: + +```bash +brew install gzip +export PAIMON_GZIP="$(brew --prefix gzip)/bin/gzip" +``` + For example, update all version locations and review the diff: ```bash @@ -80,10 +89,13 @@ The release scripts use `vVERSION-rcRC` for release-candidate tags and Start from the exact clean commit approved for the candidate. Before publishing, the wrapper fetches the release branch and requires `HEAD` to be contained in -its current history. It then creates and verifies a signed RC tag, creates the -source archive and its checksum/signature, performs the full source-release -verification, pushes the tag, waits for the tag-triggered release-candidate -workflow to succeed, and imports the artifacts into ASF `dist/dev`: +its current history. It then creates and verifies a signed RC tag, pushes the +tag, and waits for the tag-triggered release-candidate workflow. That workflow +creates the canonical source archive and checksum, builds and tests the same +archive with GCC and Clang, and uploads it as a workflow artifact. The wrapper +downloads those exact bytes, confirms that they are reproducible from the tag, +signs the archive locally, performs the full source-release verification, and +imports the three files into ASF `dist/dev`: ```bash scripts/releasing/release_rc.sh \ @@ -96,12 +108,13 @@ scripts/releasing/release_rc.sh \ The release branch defaults to `main`; use `--release-branch NAME` for a maintenance release from another Apache branch. -Use `--prepare-only` to create and verify artifacts without pushing the tag or -uploading to ASF infrastructure. This local-only mode does not require `HEAD` -to match the remote release branch. Use `--dry-run` to print identifiers -without making changes. A resumed run reuses an existing local tag or complete -artifact set only after validating it. A prepare-only run does not print a vote -email and must not be used to start a vote. +Use `--prepare-only` to create and verify preview artifacts without pushing the +tag or uploading to ASF infrastructure. This local-only mode does not require +`HEAD` to match the remote release branch. A preview is not authoritative: a +published run downloads the workflow artifact and rejects an existing local +archive if its bytes differ. Use `--dry-run` to print identifiers without +making changes. A prepare-only run does not print a vote email and must not be +used to start a vote. The candidate directory contains: @@ -164,13 +177,16 @@ The verifier checks: - installation plus compilation and execution of an external CMake consumer. Pass `--git-ref v0.3.0-rc1` when the Git repository is available to regenerate -the archive from the signed tag and compare it byte-for-byte. +the archive from the signed tag with GNU gzip and compare it byte-for-byte. +This check requires GNU gzip on every platform; it intentionally rejects the +macOS system gzip instead of treating different compressed bytes as equivalent. `--allow-unsigned`, `--skip-rat`, `--skip-build`, and `--skip-install` exist for CI or local development of the release process. They are not a substitute for the corresponding checks when voting. The release-candidate workflow creates -an unsigned archive for deterministic CI validation; official artifacts must -always be signed by the release manager. +the unsigned canonical archive for deterministic CI validation. The release +manager downloads and signs that exact archive; the private signing key remains +only on the release manager's machine. ## Publish an approved release @@ -207,8 +223,9 @@ than ASF's general one-hour minimum. - `bump_version.py`: consistently check or update CMake and documentation version metadata. -- `create_source_release.sh`: deterministically create an archive, SHA-512 - checksum, and optional detached signature from an immutable Git ref. +- `create_source_release.sh`: deterministically create an archive with GNU + gzip, a SHA-512 checksum, and an optional detached signature from an immutable + Git ref. - `validate_source_archive.py`: reject unsafe or non-portable tar members and compiled files. - `verify_release_candidate.sh`: perform voter-facing integrity, license, diff --git a/scripts/releasing/create_source_release.sh b/scripts/releasing/create_source_release.sh index 125ab579..67e4d093 100755 --- a/scripts/releasing/create_source_release.sh +++ b/scripts/releasing/create_source_release.sh @@ -49,6 +49,9 @@ The script creates: apache-paimon-cpp-VERSION-src.tgz.asc (when --signing-key is provided) Existing artifacts are never overwritten. + +GNU gzip is required so macOS and Linux produce the same compressed bytes. +Set PAIMON_GZIP to an explicit GNU gzip executable when it is not on PATH. EOF } @@ -68,6 +71,36 @@ calculate_sha512() { fi } +find_gnu_gzip() { + local candidate + local resolved + local version_line + local -a candidates + + if [[ -n "${PAIMON_GZIP:-}" ]]; then + candidates=("${PAIMON_GZIP}") + else + candidates=(gzip ggzip) + fi + + for candidate in "${candidates[@]}"; do + if [[ -x "${candidate}" ]]; then + resolved=${candidate} + elif resolved=$(command -v "${candidate}" 2>/dev/null); then + : + else + continue + fi + version_line=$("${resolved}" --version 2>/dev/null | sed -n '1p' || true) + if [[ "${version_line}" =~ ^gzip[[:space:]][0-9] ]]; then + printf '%s\n' "${resolved}" + return 0 + fi + done + + fail "GNU gzip is required for reproducible source archives; on macOS run 'brew install gzip' and set PAIMON_GZIP to the Homebrew gzip executable" +} + while [[ $# -gt 0 ]]; do case "$1" in --version) @@ -128,6 +161,8 @@ DOCS_VERSION=$( [[ "${DOCS_VERSION}" == "${RELEASE_VERSION}" ]] || fail "documentation version ${DOCS_VERSION:-} does not match ${RELEASE_VERSION}" +GZIP_BIN=$(find_gnu_gzip) + ARTIFACT_NAME="apache-paimon-cpp-${RELEASE_VERSION}-src.tgz" ARCHIVE_ROOT="paimon-cpp-${RELEASE_VERSION}" @@ -147,7 +182,10 @@ git -C "${SOURCE_ROOT}" -c tar.umask=0022 archive \ --format=tar \ --prefix="${ARCHIVE_ROOT}/" \ "${GIT_REF}" | - gzip -n >"${TEMP_DIR}/${ARTIFACT_NAME}" + ( + unset GZIP + "${GZIP_BIN}" --no-name --stdout -6 + ) >"${TEMP_DIR}/${ARTIFACT_NAME}" SHA512=$(calculate_sha512 "${TEMP_DIR}/${ARTIFACT_NAME}") printf '%s %s\n' "${SHA512}" "${ARTIFACT_NAME}" \ diff --git a/scripts/releasing/release_rc.sh b/scripts/releasing/release_rc.sh index f943491e..34548638 100755 --- a/scripts/releasing/release_rc.sh +++ b/scripts/releasing/release_rc.sh @@ -31,6 +31,8 @@ DIST_DEV_BASE_URL="https://dist.apache.org/repos/dist/dev/paimon" PREPARE_ONLY=false DRY_RUN=false WORKFLOW_DISCOVERY_TIMEOUT_SECONDS=600 +WORKFLOW_RUN_ID="" +TEMP_DIR="" usage() { cat <<'EOF' @@ -53,8 +55,9 @@ Options: --dry-run Print the planned release identifiers and exit -h, --help Show this help -The script is resumable when the local signed tag or artifacts already exist, -provided that they match HEAD and pass all verification checks. +For a published RC, GitHub Actions creates and tests the canonical source +archive. This script downloads those exact bytes, signs them locally, and +uploads them to ASF dist/dev. Prepare-only mode creates a local preview. EOF } @@ -117,6 +120,74 @@ wait_for_release_candidate_workflow() { --exit-status \ --interval 30 || fail "Release Candidate workflow run ${run_id} failed" + WORKFLOW_RUN_ID=${run_id} +} + +validate_workflow_artifact_directory() { + local directory=$1 + local -a entries + local entry + local name + + shopt -s dotglob nullglob + entries=("${directory}"/*) + shopt -u dotglob nullglob + [[ ${#entries[@]} -eq 2 ]] || + fail "workflow artifact must contain exactly the archive and checksum" + for entry in "${entries[@]}"; do + [[ -f "${entry}" ]] || + fail "workflow artifact contains a non-file entry: ${entry}" + name=$(basename "${entry}") + case "${name}" in + "${ARTIFACT_NAME}" | "${ARTIFACT_NAME}.sha512") + ;; + *) + fail "workflow artifact contains an unexpected file: ${name}" + ;; + esac + done +} + +download_and_sign_workflow_artifact() { + local workflow_dir="${TEMP_DIR}/source-archive" + local source + local target + local suffix + + [[ -n "${WORKFLOW_RUN_ID}" ]] || fail "release workflow run ID is missing" + mkdir -p "${workflow_dir}" + gh run download "${WORKFLOW_RUN_ID}" \ + --repo apache/paimon-cpp \ + --name source-archive \ + --dir "${workflow_dir}" + validate_workflow_artifact_directory "${workflow_dir}" + + mkdir -p "${OUTPUT_DIR}" + for suffix in "" ".sha512"; do + source="${workflow_dir}/${ARTIFACT_NAME}${suffix}" + target="${OUTPUT_DIR}/${ARTIFACT_NAME}${suffix}" + if [[ -e "${target}" ]]; then + [[ -f "${target}" ]] || fail "artifact path is not a file: ${target}" + cmp "${source}" "${target}" >/dev/null || + fail "existing ${target} differs from workflow run ${WORKFLOW_RUN_ID}" + else + cp -p "${source}" "${target}" + fi + done + + ARTIFACT="${OUTPUT_DIR}/${ARTIFACT_NAME}" + if [[ -e "${ARTIFACT}.asc" ]]; then + [[ -f "${ARTIFACT}.asc" ]] || + fail "artifact signature path is not a file: ${ARTIFACT}.asc" + echo "Reusing existing source artifact signature." + else + echo "Signing workflow source artifact with ${SIGNING_KEY}." + gpg --armor \ + --local-user "${SIGNING_KEY}" \ + --detach-sign \ + --output "${ARTIFACT}.asc" \ + "${ARTIFACT}" + fi } validate_artifact_directory() { @@ -224,6 +295,9 @@ EOF exit 0 fi +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "${TEMP_DIR}"' EXIT + for command in git gpg python3; do require_command "${command}" done @@ -269,33 +343,33 @@ else fi ARTIFACT="${OUTPUT_DIR}/${ARTIFACT_NAME}" -if [[ -e "${ARTIFACT}" || -e "${ARTIFACT}.asc" || -e "${ARTIFACT}.sha512" ]]; then - [[ -f "${ARTIFACT}" && -f "${ARTIFACT}.asc" && -f "${ARTIFACT}.sha512" ]] || - fail "artifact directory contains an incomplete release candidate" - echo "Reusing existing artifacts in ${OUTPUT_DIR}." -else - "${SCRIPT_DIR}/create_source_release.sh" \ - --version "${VERSION}" \ +if [[ "${PREPARE_ONLY}" == true ]]; then + if [[ -e "${ARTIFACT}" || -e "${ARTIFACT}.asc" || -e "${ARTIFACT}.sha512" ]]; then + [[ -f "${ARTIFACT}" && -f "${ARTIFACT}.asc" && -f "${ARTIFACT}.sha512" ]] || + fail "artifact directory contains an incomplete release candidate" + echo "Reusing existing preview artifacts in ${OUTPUT_DIR}." + else + "${SCRIPT_DIR}/create_source_release.sh" \ + --version "${VERSION}" \ + --git-ref "${RC_TAG}" \ + --output-dir "${OUTPUT_DIR}" \ + --signing-key "${SIGNING_KEY}" + fi + + "${SCRIPT_DIR}/verify_release_candidate.sh" \ --git-ref "${RC_TAG}" \ - --output-dir "${OUTPUT_DIR}" \ - --signing-key "${SIGNING_KEY}" -fi - -"${SCRIPT_DIR}/verify_release_candidate.sh" \ - --git-ref "${RC_TAG}" \ - --keys-url "https://downloads.apache.org/paimon/KEYS" \ - "${ARTIFACT}" + --keys-url "https://downloads.apache.org/paimon/KEYS" \ + "${ARTIFACT}" + validate_artifact_directory -validate_artifact_directory - -if [[ "${PREPARE_ONLY}" == true ]]; then cat </dev/null 2>&1; then fi git push "${REMOTE}" "${RC_TAG}" wait_for_release_candidate_workflow +download_and_sign_workflow_artifact + +"${SCRIPT_DIR}/verify_release_candidate.sh" \ + --git-ref "${RC_TAG}" \ + --keys-url "https://downloads.apache.org/paimon/KEYS" \ + "${ARTIFACT}" +validate_artifact_directory + svn import "${OUTPUT_DIR}" "${RC_URL}" \ -m "Add Apache Paimon C++ ${VERSION} RC${RC}" diff --git a/scripts/releasing/tests/test_release_tools.py b/scripts/releasing/tests/test_release_tools.py index d1b4eeaf..827ddfea 100644 --- a/scripts/releasing/tests/test_release_tools.py +++ b/scripts/releasing/tests/test_release_tools.py @@ -20,6 +20,8 @@ import io import json import os +import re +import shutil import subprocess import sys import tarfile @@ -33,9 +35,26 @@ ARCHIVE_VALIDATOR = RELEASING_DIR / "validate_source_archive.py" VERSION_TOOL = RELEASING_DIR / "bump_version.py" RELEASE_VERIFIER = RELEASING_DIR / "verify_release_candidate.sh" +SOURCE_RELEASE_CREATOR = RELEASING_DIR / "create_source_release.sh" +SOURCE_ROOT = RELEASING_DIR.parents[1] class ReleaseToolTest(unittest.TestCase): + def head_release_version(self) -> str: + result = subprocess.run( + ["git", "-C", str(SOURCE_ROOT), "show", "HEAD:CMakeLists.txt"], + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + self.assertEqual(result.returncode, 0, msg=result.stderr) + match = re.search( + r"^\s*VERSION\s+(\d+\.\d+\.\d+)\s*$", result.stdout, re.MULTILINE + ) + self.assertIsNotNone(match) + return match.group(1) + def run_tool( self, tool: Path, *args: str, expected_returncode: int = 0 ) -> subprocess.CompletedProcess: @@ -314,6 +333,269 @@ def test_verifier_rejects_unknown_licenses(self) -> None: result.stderr, ) + @unittest.skipUnless( + shutil.which("gpg") and shutil.which("gzip"), "gpg and gzip are required" + ) + def test_verifier_uses_keys_file_for_unsigned_artifact_tag(self) -> None: + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + source_root = directory / "source" + releasing_dir = source_root / "scripts/releasing" + releasing_dir.mkdir(parents=True) + for script in ( + "bump_version.py", + "create_source_release.sh", + "validate_source_archive.py", + "verify_release_candidate.sh", + ): + shutil.copy2(RELEASING_DIR / script, releasing_dir / script) + + files = { + "CMakeLists.txt": "project(paimon\n VERSION 1.2.3\n)\n", + "LICENSE": "Apache License\n", + "NOTICE": "Apache Paimon\n", + "docs/source/conf.py": 'version = "1.2.3"\n', + "docs/source/_static/versions.json": ( + '[{"name": "1.2.3", "version": "1.2.3", ' + '"url": "https://paimon.apache.org/docs/cpp/"}]\n' + ), + ".github/.rat-excludes": "", + } + for name, content in files.items(): + path = source_root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + empty_git_config = directory / "empty-gitconfig" + empty_git_config.touch() + git_env = os.environ.copy() + for name in list(git_env): + if name in ("GIT_CONFIG_COUNT", "GIT_CONFIG_PARAMETERS") or re.fullmatch( + r"GIT_CONFIG_(KEY|VALUE)_\d+", name + ): + del git_env[name] + git_env["GIT_CONFIG_GLOBAL"] = str(empty_git_config) + git_env["GIT_CONFIG_SYSTEM"] = str(empty_git_config) + + subprocess.run( + ["git", "init", "-q", str(source_root)], env=git_env, check=True + ) + subprocess.run( + ["git", "-C", str(source_root), "config", "user.name", "Release Test"], + env=git_env, + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(source_root), + "config", + "user.email", + "release-test@example.com", + ], + env=git_env, + check=True, + ) + subprocess.run( + ["git", "-C", str(source_root), "add", "."], + env=git_env, + check=True, + ) + subprocess.run( + ["git", "-C", str(source_root), "commit", "-q", "-m", "test"], + env=git_env, + check=True, + ) + + signing_home = directory / "signing-home" + signing_home.mkdir(mode=0o700) + signing_env = git_env.copy() + signing_env["GNUPGHOME"] = str(signing_home) + subprocess.run( + [ + "gpg", + "--batch", + "--passphrase", + "", + "--quick-generate-key", + "Release Test ", + "ed25519", + "sign", + "0", + ], + env=signing_env, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + key_listing = subprocess.run( + ["gpg", "--batch", "--with-colons", "--list-secret-keys"], + env=signing_env, + universal_newlines=True, + stdout=subprocess.PIPE, + check=True, + ) + fingerprint = next( + line.split(":")[9] + for line in key_listing.stdout.splitlines() + if line.startswith("fpr:") # codespell:ignore fpr + ) + subprocess.run( + [ + "git", + "-C", + str(source_root), + "tag", + "-s", + "-u", + fingerprint, + "-m", + "test tag", + "v1.2.3-rc1", + ], + env=signing_env, + check=True, + ) + + keys_file = directory / "KEYS" + with keys_file.open("w", encoding="utf-8") as output: + subprocess.run( + ["gpg", "--batch", "--armor", "--export", fingerprint], + env=signing_env, + universal_newlines=True, + stdout=output, + check=True, + ) + + real_gzip = shutil.which("gzip") + self.assertIsNotNone(real_gzip) + fake_gzip = directory / "gzip" + fake_gzip.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "if [[ ${1:-} == --version ]]; then\n" + " echo 'gzip 1.99'\n" + " exit 0\n" + "fi\n" + f'exec "{real_gzip}" -n -c -6\n', + encoding="utf-8", + ) + fake_gzip.chmod(0o755) + release_env = git_env.copy() + release_env["PAIMON_GZIP"] = str(fake_gzip) + + artifact_dir = directory / "release" + subprocess.run( + [ + "bash", + str(releasing_dir / "create_source_release.sh"), + "--version", + "1.2.3", + "--git-ref", + "v1.2.3-rc1", + "--output-dir", + str(artifact_dir), + ], + env=release_env, + check=True, + stdout=subprocess.DEVNULL, + ) + artifact = artifact_dir / "apache-paimon-cpp-1.2.3-src.tgz" + result = subprocess.run( + [ + "bash", + str(releasing_dir / "verify_release_candidate.sh"), + "--allow-unsigned", + "--keys-file", + str(keys_file), + "--git-ref", + "v1.2.3-rc1", + "--skip-rat", + "--skip-build", + str(artifact), + ], + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=release_env, + check=False, + ) + self.assertEqual( + result.returncode, 0, msg=result.stdout + result.stderr + ) + self.assertIn("Git ref reproducibility: valid", result.stdout) + + def test_source_creator_rejects_non_gnu_gzip(self) -> None: + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + fake_gzip = directory / "gzip" + fake_gzip.write_text( + "#!/usr/bin/env bash\n" + "echo 'Apple gzip 999.0'\n", + encoding="utf-8", + ) + fake_gzip.chmod(0o755) + env = os.environ.copy() + env["PAIMON_GZIP"] = str(fake_gzip) + result = subprocess.run( + [ + "bash", + str(SOURCE_RELEASE_CREATOR), + "--version", + self.head_release_version(), + "--git-ref", + "HEAD", + "--output-dir", + str(directory / "release"), + ], + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + ) + self.assertEqual(result.returncode, 1, msg=result.stdout + result.stderr) + self.assertIn("GNU gzip is required", result.stderr) + + def test_source_creator_clears_gzip_environment_options(self) -> None: + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + fake_gzip = directory / "gzip" + fake_gzip.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "if [[ ${1:-} == --version ]]; then\n" + " echo 'gzip 1.99'\n" + " exit 0\n" + "fi\n" + "[[ -z ${GZIP+x} ]] || { echo 'GZIP was not cleared' >&2; exit 1; }\n" + "dd of=/dev/null 2>/dev/null\n", + encoding="utf-8", + ) + fake_gzip.chmod(0o755) + env = os.environ.copy() + env["GZIP"] = "-9" + env["PAIMON_GZIP"] = str(fake_gzip) + result = subprocess.run( + [ + "bash", + str(SOURCE_RELEASE_CREATOR), + "--version", + self.head_release_version(), + "--git-ref", + "HEAD", + "--output-dir", + str(directory / "release"), + ], + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + ) + self.assertEqual(result.returncode, 0, msg=result.stdout + result.stderr) + def create_version_tree(self, root: Path) -> None: (root / "docs/source/_static").mkdir(parents=True) (root / "CMakeLists.txt").write_text( diff --git a/scripts/releasing/verify_release_candidate.sh b/scripts/releasing/verify_release_candidate.sh index 99e7df0a..0f3e76d0 100755 --- a/scripts/releasing/verify_release_candidate.sh +++ b/scripts/releasing/verify_release_candidate.sh @@ -28,6 +28,8 @@ DIST_DEV_BASE_URL="https://dist.apache.org/repos/dist/dev/paimon" KEYS_URL="" KEYS_FILE="" GIT_REF="" +VERIFY_GNUPG_HOME="" +VERIFY_KEYS_FILE="" RAT_JAR=${RAT_JAR:-} RAT_VERSION="0.16.1" ALLOW_UNSIGNED=false @@ -52,7 +54,7 @@ Download options: Trust and reproducibility: --keys-url URL Download KEYS and verify in an isolated GPG home --keys-file FILE Import this KEYS file into an isolated GPG home - --git-ref REF Regenerate the archive from REF and compare bytes + --git-ref REF Regenerate with GNU gzip and compare archive bytes Verification options: --rat-jar FILE Apache RAT executable jar (or set RAT_JAR) @@ -99,6 +101,24 @@ download_file() { fi } +prepare_verification_keyring() { + [[ -z "${VERIFY_GNUPG_HOME}" ]] || return 0 + + command -v gpg >/dev/null 2>&1 || fail "gpg is required for signature verification" + VERIFY_GNUPG_HOME="${TEMP_DIR}/gnupg" + mkdir -m 700 "${VERIFY_GNUPG_HOME}" + if [[ -n "${KEYS_URL}" ]]; then + VERIFY_KEYS_FILE="${TEMP_DIR}/KEYS" + download_file "${KEYS_URL}" "${VERIFY_KEYS_FILE}" + else + VERIFY_KEYS_FILE=$(cd "$(dirname "${KEYS_FILE}")" && pwd)/$(basename "${KEYS_FILE}") + fi + [[ -f "${VERIFY_KEYS_FILE}" ]] || + fail "KEYS file does not exist: ${VERIFY_KEYS_FILE}" + gpg --batch --homedir "${VERIFY_GNUPG_HOME}" \ + --import "${VERIFY_KEYS_FILE}" >/dev/null +} + while [[ $# -gt 0 ]]; do case "$1" in --version) @@ -249,19 +269,10 @@ echo "SHA-512 checksum: valid" if [[ -f "${SIGNATURE_FILE}" ]]; then command -v gpg >/dev/null 2>&1 || fail "gpg is required to verify the signature" if [[ -n "${KEYS_URL}" || -n "${KEYS_FILE}" ]]; then - GNUPG_HOME="${TEMP_DIR}/gnupg" - mkdir -m 700 "${GNUPG_HOME}" - if [[ -n "${KEYS_URL}" ]]; then - KEYS_FILE="${TEMP_DIR}/KEYS" - download_file "${KEYS_URL}" "${KEYS_FILE}" - else - KEYS_FILE=$(cd "$(dirname "${KEYS_FILE}")" && pwd)/$(basename "${KEYS_FILE}") - fi - [[ -f "${KEYS_FILE}" ]] || fail "KEYS file does not exist: ${KEYS_FILE}" - gpg --batch --homedir "${GNUPG_HOME}" --import "${KEYS_FILE}" >/dev/null - gpg --batch --homedir "${GNUPG_HOME}" \ + prepare_verification_keyring + gpg --batch --homedir "${VERIFY_GNUPG_HOME}" \ --verify "${SIGNATURE_FILE}" "${ARTIFACT}" - echo "OpenPGP signature: valid against ${KEYS_FILE}" + echo "OpenPGP signature: valid against ${VERIFY_KEYS_FILE}" else gpg --verify "${SIGNATURE_FILE}" "${ARTIFACT}" echo "OpenPGP signature: valid against the default GPG keyring" @@ -313,7 +324,13 @@ if [[ -n "${GIT_REF}" ]]; then fail "Git ref does not resolve to a commit: ${GIT_REF}" if git -C "${SOURCE_ROOT}" rev-parse --verify "${GIT_REF}^{tag}" \ >/dev/null 2>&1; then - git -C "${SOURCE_ROOT}" verify-tag "${GIT_REF}" + if [[ -n "${KEYS_URL}" || -n "${KEYS_FILE}" ]]; then + prepare_verification_keyring + GNUPGHOME=${VERIFY_GNUPG_HOME} \ + git -C "${SOURCE_ROOT}" verify-tag "${GIT_REF}" + else + git -C "${SOURCE_ROOT}" verify-tag "${GIT_REF}" + fi fi REPRO_DIR="${TEMP_DIR}/reproduced" "${SCRIPT_DIR}/create_source_release.sh" \ From 9d19960325a4ccc8dad35587b180095861aa68f3 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:49:12 +0800 Subject: [PATCH 2/6] fix: support nullable map key schemas and reject null key data (#176) * fix: support nullable map key schemas and reject null key data * add validate for avro --- docs/source/user_guide/data_types.rst | 7 ++- docs/source/user_guide/schema.rst | 32 ++----------- .../common/types/data_type_json_parser.cpp | 14 ++---- .../types/data_type_json_parser_test.cpp | 4 +- .../core/schema/arrow_schema_validator.cpp | 4 -- .../core/schema/schema_validation_test.cpp | 10 ++-- src/paimon/core/schema/table_schema_test.cpp | 17 ++++--- .../format/avro/avro_file_batch_reader.cpp | 1 + src/paimon/format/orc/orc_adapter.cpp | 1 + .../parquet/parquet_file_batch_reader.cpp | 1 + test/inte/scan_and_read_inte_test.cpp | 44 ++++++++++++++++++ .../append_multiple/schema/schema-0 | 2 +- .../append_simple/schema/schema-0 | 2 +- .../append_with_multiple_map/schema/schema-0 | 18 +++---- .../pk_with_multiple_type/schema/schema-0 | 2 +- .../schema/schema-0 | 2 +- .../schema/schema-0 | 2 +- .../nullable_map_key/README | 13 ++++++ ...7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc | Bin 0 -> 562 bytes ...est-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0 | Bin 0 -> 2089 bytes ...ist-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0 | Bin 0 -> 1006 bytes ...ist-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1 | Bin 0 -> 1110 bytes .../nullable_map_key/schema/schema-0 | 25 ++++++++++ .../nullable_map_key/snapshot/EARLIEST | 1 + .../nullable_map_key/snapshot/LATEST | 1 + .../nullable_map_key/snapshot/snapshot-1 | 17 +++++++ .../pk_table_nested_type/schema/schema-0 | 2 +- .../schema/schema-0 | 2 +- .../parquet_append_table/schema/schema-0 | 2 +- .../pk_table_nested_type/schema/schema-0 | 2 +- 30 files changed, 154 insertions(+), 74 deletions(-) create mode 100644 test/test_data/orc/nullable_map_key.db/nullable_map_key/README create mode 100644 test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc create mode 100644 test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0 create mode 100644 test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0 create mode 100644 test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1 create mode 100644 test/test_data/orc/nullable_map_key.db/nullable_map_key/schema/schema-0 create mode 100644 test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/EARLIEST create mode 100644 test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/LATEST create mode 100644 test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/snapshot-1 diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index f5326428..3d529332 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -188,15 +188,14 @@ and `Arrow DataTypes `` - Map - - Data type of an associative array that maps keys to values (including NULL). A map cannot contain duplicate keys; each key can map to at most one value. + - Data type of an associative array that maps keys (including NULL) to values (including NULL). A map cannot contain duplicate keys; each key can map to at most one value. There is no restriction of element types; it is the responsibility of the user to ensure uniqueness. The type can be declared using ``MAP`` where kt is the data type of the key elements and vt is the data type of the value elements. - **Note:** In Paimon C++, map keys must be explicitly marked as ``NOT NULL``. - Apache Arrow does not support nullable map keys. If the key type is not - marked as ``NOT NULL`` in the schema, parsing will fail with an error. + **Note:** Paimon C++ accepts nullable map key declarations for schema compatibility. + A query fails only if the data actually contains a NULL map key. * - ``MULTISET`` - Not Supported diff --git a/docs/source/user_guide/schema.rst b/docs/source/user_guide/schema.rst index 0fe8e720..5db69ef9 100644 --- a/docs/source/user_guide/schema.rst +++ b/docs/source/user_guide/schema.rst @@ -84,35 +84,11 @@ DataField represents a column of the table. 3. ``type``: data type, very similar to SQL type string. 4. ``description``: string. -Limitations ------------ +Nullable MAP Keys +----------------- -MAP Key Must Be NOT NULL -^^^^^^^^^^^^^^^^^^^^^^^^ - -Apache Arrow does not support nullable map keys. When defining a ``MAP`` type in the schema, -the key must be explicitly marked as ``NOT NULL``. If the key is not marked as ``NOT NULL``, -schema parsing will fail with an error. - -For example, the following is **valid**: - -.. code-block:: json - - { - "type": "MAP", - "key": "TINYINT NOT NULL", - "value": "SMALLINT" - } - -The following is **invalid** and will be rejected: - -.. code-block:: json - - { - "type": "MAP", - "key": "TINYINT", - "value": "SMALLINT" - } +Paimon C++ accepts nullable ``MAP`` key declarations for compatibility with existing schemas. +A query fails only when the data actually contains a NULL map key. Update Schema ------------- diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index 950308a5..33308ab6 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -689,17 +689,13 @@ Result> DataTypeJsonParser::ParseMapType( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key, ParseType("key", type_json_value["key"])); + // NOTE: Unlike Java Paimon, this C++ implementation does not support nullable keys in // MapType. This is a limitation of Apache Arrow, which does not allow null keys in its - // MapType. As a result, we validate `nullable = false` for the map key. - if (key->nullable()) { - return Status::Invalid(fmt::format( - "Map field '{}' has a nullable key." - "Map keys must be explicitly marked as NOT NULL in the schema for paimon-cpp " - "because Apache Arrow does not support nullable map keys. " - "Please add 'NOT NULL' to the key type definition.", - name)); - } + // MapType. As a result, we explicitly set `nullable = false` for the map key, regardless of + // the original schema. Please be aware of this behavioral difference when migrating or + // interoperating with Java Paimon. + key = key->WithNullable(false); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr value, ParseType("value", type_json_value["value"])); return arrow::field(name, std::make_shared(key, value), nullable); diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index bbb4fb52..bfa69c32 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -51,7 +51,7 @@ TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) { const std::string name = "map_field"; const char* json = R"({ "type": "MAP", - "key": "STRING NOT NULL", + "key": "STRING", "value": "INT" })"; rapidjson::Document doc; @@ -60,6 +60,8 @@ TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) { ASSERT_OK_AND_ASSIGN(std::shared_ptr field, DataTypeJsonParser::ParseType(name, doc)); ASSERT_NE(field, nullptr); + auto map_type = std::static_pointer_cast(field->type()); + ASSERT_FALSE(map_type->key_field()->nullable()); } TEST(DataTypeJsonParserTest, ParseTypeRowTypeSuccess) { diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index b9ea5da2..e01afe01 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -231,10 +231,6 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& arrow::internal::checked_cast(*field->type()).key_field(); const auto& item_field = arrow::internal::checked_cast(*field->type()).item_field(); - if (key_field->nullable()) { - return Status::Invalid( - fmt::format("Map field '{}' has a nullable key.", field->name())); - } PAIMON_RETURN_NOT_OK(ValidateField(key_field, /*allow_blob=*/false)); PAIMON_RETURN_NOT_OK(ValidateField(item_field, /*allow_blob=*/false)); break; diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 57179e8c..3c3054ac 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -938,7 +938,7 @@ TEST(SchemaValidationTest, TestMapStorageLayout) { } } -TEST(SchemaValidationTest, TestMapRequiresNonNullableKey) { +TEST(SchemaValidationTest, TestMapSharedShreddingRequiresNonNullableKey) { auto nullable_key_map = std::make_shared(arrow::field("key", arrow::utf8(), /*nullable=*/true), arrow::field("value", arrow::int64())); @@ -951,9 +951,11 @@ TEST(SchemaValidationTest, TestMapRequiresNonNullableKey) { {Options::BUCKET_KEY, "f0"}, {"fields.f1.map.storage-layout", "shared-shredding"}, }; - ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options), - "Map field 'f1' has a nullable key."); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "map key type is nullable"); } TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) { diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 988f17f9..dba7d833 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -1258,7 +1258,7 @@ TEST_F(TableSchemaTest, SetFieldIdNestedListInStruct) { } } -TEST_F(TableSchemaTest, MapKeyMustBeNotNull) { +TEST_F(TableSchemaTest, NullableMapKeySchemaIsSupported) { std::string table_schema_str = R"({ "version" : 3, "id" : 0, @@ -1277,16 +1277,21 @@ TEST_F(TableSchemaTest, MapKeyMustBeNotNull) { "options" : {}, "timeMillis" : 1721614341162 })"; - ASSERT_NOK_WITH_MSG(TableSchema::CreateFromJson(table_schema_str), - "Map field 'f0' has a nullable key."); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_schema, + TableSchema::CreateFromJson(table_schema_str)); + auto json_map_type = std::static_pointer_cast(table_schema->Fields()[0].Type()); + ASSERT_FALSE(json_map_type->key_field()->nullable()); auto nullable_key_map = std::make_shared(arrow::field("key", arrow::int8(), /*nullable=*/true), arrow::field("value", arrow::int16())); - ASSERT_NOK_WITH_MSG( + ASSERT_OK_AND_ASSIGN( + std::shared_ptr direct_table_schema, TableSchema::Create(/*schema_id=*/0, arrow::schema({arrow::field("f0", nullable_key_map)}), - /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{}), - "Map field 'f0' has a nullable key."); + /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); + auto direct_map_type = + std::static_pointer_cast(direct_table_schema->Fields()[0].Type()); + ASSERT_TRUE(direct_map_type->key_field()->nullable()); } TEST_F(TableSchemaTest, MapKeysSortedIsNormalized) { diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp index 92ac769c..f48ec4cc 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader.cpp @@ -123,6 +123,7 @@ Result AvroFileBatchReader::NextBatch() { } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, array_builder_->Finish()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); diff --git a/src/paimon/format/orc/orc_adapter.cpp b/src/paimon/format/orc/orc_adapter.cpp index 992387d1..4773f2ed 100644 --- a/src/paimon/format/orc/orc_adapter.cpp +++ b/src/paimon/format/orc/orc_adapter.cpp @@ -945,6 +945,7 @@ Result> OrcAdapter::AppendBatch( MakeArrowBuilder(type, batch, pool)); std::shared_ptr array; PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); return array; } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 4f1d013f..fcc47c18 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -537,6 +537,7 @@ Result ParquetFileBatchReader::NextBatch() { } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, batch->ToStructArray()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( array->type(), read_data_type_)); if (need_cast) { diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index a9294ef0..23384632 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -2819,6 +2819,50 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } +TEST_P(ScanAndReadInteTest, TestReadNullableMapKey) { + auto file_format = FileFormat(); + if (file_format != "orc") { + return; + } + // Java Parquet does not support writing null MAP keys; see + // ParquetRowDataWriter.MapWriter#writeMapData in Apache Paimon. Therefore, this test uses an + // ORC table. The table contains three rows: the first two rows have non-null MAP keys, while + // the third row has a null MAP key and is expected to fail during reading. + const std::string table_path = + paimon::test::GetDataDir() + "orc/nullable_map_key.db/nullable_map_key"; + + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_TRUE(result_plan->SnapshotId()); + ASSERT_EQ(result_plan->SnapshotId().value(), 1); + ASSERT_EQ(result_plan->Splits().size(), 1); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.AddOption(Options::READ_BATCH_SIZE, "1"); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + + const std::vector expected_rows = {R"([[0, 1, [["one", 10]]]])", + R"([[0, 2, [["two", 20]]]])"}; + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_EQ(array->length(), 1); + + std::shared_ptr expected_array = + arrow::ipc::internal::json::ArrayFromJSON(array->type(), expected_rows[i]).ValueOrDie(); + ASSERT_TRUE(array->Equals(expected_array)) + << "actual: " << array->ToString() << ", expected: " << expected_array->ToString(); + } + ASSERT_NOK_WITH_MSG(batch_reader->NextBatch(), "Map array keys array should have no nulls"); + batch_reader->Close(); +} + TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + diff --git a/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0 b/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0 index e3b8f71a..c0d6d755 100644 --- a/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0 +++ b/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0 @@ -93,7 +93,7 @@ "name": "f0", "type": { "type": "MAP", - "key": "STRING NOT NULL", + "key": "STRING", "value": "INT" } }, diff --git a/test/test_data/avro/append_simple.db/append_simple/schema/schema-0 b/test/test_data/avro/append_simple.db/append_simple/schema/schema-0 index c8b4be27..be12e26c 100644 --- a/test/test_data/avro/append_simple.db/append_simple/schema/schema-0 +++ b/test/test_data/avro/append_simple.db/append_simple/schema/schema-0 @@ -23,7 +23,7 @@ "name" : "f0", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : "INT" } }, { diff --git a/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0 b/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0 index 72376320..b118e224 100644 --- a/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0 +++ b/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0 @@ -6,7 +6,7 @@ "name" : "f0", "type" : { "type" : "MAP", - "key" : "INT NOT NULL", + "key" : "INT", "value" : "INT" } }, { @@ -14,7 +14,7 @@ "name" : "f1", "type" : { "type" : "MAP", - "key" : "DOUBLE NOT NULL", + "key" : "DOUBLE", "value" : "DOUBLE" } }, { @@ -22,7 +22,7 @@ "name" : "f2", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : "STRING" } }, { @@ -30,7 +30,7 @@ "name" : "f3", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : "BINARY(6)" } }, { @@ -38,7 +38,7 @@ "name" : "f4", "type" : { "type" : "MAP", - "key" : "TIMESTAMP(6) NOT NULL", + "key" : "TIMESTAMP(6)", "value" : "TIMESTAMP(6)" } }, { @@ -46,7 +46,7 @@ "name" : "f5", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : { "type" : "ARRAY", "element" : "DOUBLE" @@ -57,10 +57,10 @@ "name" : "f6", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : { "type" : "MAP", - "key" : "DOUBLE NOT NULL", + "key" : "DOUBLE", "value" : "STRING" } } @@ -69,7 +69,7 @@ "name" : "f7", "type" : { "type" : "MAP", - "key" : "BIGINT NOT NULL", + "key" : "BIGINT", "value" : { "type" : "ROW", "fields" : [ { diff --git a/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0 b/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0 index 5533e14b..608d0df8 100644 --- a/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0 +++ b/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0 @@ -55,7 +55,7 @@ "name" : "f0", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : "INT" } }, { diff --git a/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 b/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 index 699682f2..f973a64c 100644 --- a/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 +++ b/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 @@ -6,7 +6,7 @@ "name" : "f1", "type" : { "type" : "MAP", - "key" : "TINYINT NOT NULL", + "key" : "TINYINT", "value" : "SMALLINT" } }, { diff --git a/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0 b/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0 index c754fec8..57cd38af 100644 --- a/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0 +++ b/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0 @@ -64,7 +64,7 @@ "type" : { "type" : "MAP", "key" : { - "type" : "ROW NOT NULL", + "type" : "ROW", "fields" : [ { "id" : 13, "name" : "sub1", diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/README b/test/test_data/orc/nullable_map_key.db/nullable_map_key/README new file mode 100644 index 00000000..02b0d4e9 --- /dev/null +++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/README @@ -0,0 +1,13 @@ +id:int attrs:map(string, int) (all can be null) +no partition key +no primary key +bucket count: -1 +file format: orc + +Rows (snapshot-1): +Add: [1, {"one": 10}] +Add: [2, {"two": 20}] +Add: [3, {null: 30}] +NoCompact + +The third row contains a null MAP key and is expected to fail during reading. diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc b/test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..89c6f60a7fec108fe1629b5877fc76807fb7c1e9 GIT binary patch literal 562 zcmeYdau#G@;9?VE;b0D6&;~MvxtJLk7=(B@n1$Flm;~4)cmfy{i2 z)X4}W{vY6FU;yd^>SAYLaAN$x&A{NtB%)!ngMmR~tNvewoViR4SD3ljIM@UjB)Gve zql5&Lgfz2+3JaqEqXaupj7fk=f)z+J2rx4WFiUVUOR%v>a0oGRFaR--wEkv%`-bMP zQ>}A-86+RHZB6A|4D^CO7dQa8_>jFF!OkTJ@@@nxl83oOFg+L{0@VohSOiB6BgDzi z*RnDwacgQAWUpa9khSEXY-XL_LbIa>scb5n9OD= zWp-{k!^NY-F;B8=0<%C<(P95E-Z1wT&#UJbzq{~PwfY>V?EuZ z6YR9}<*e@a+_!p}8`^3b{-#@T+~8+nU<;4@VjO60$-HfL8plCK0al5I1_31|1|E$N O1J0%|%*_5l&JqACm5N0G literal 0 HcmV?d00001 diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0 b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0 new file mode 100644 index 0000000000000000000000000000000000000000..e96a4bbe52dd0fa5e619cea547c4aac021f866f8 GIT binary patch literal 2089 zcmdT_O=#0#816P54piI}C+H@8OeffN18)-#S^JHKCFz!*ZaPZLlB|nM(wb(%Hj2nR z2)np}%#%!LeC8DOO=_xT|fU*M`?&z!NRk+Z&OQtajERS!pJ^p zMHE!l$xBnw9V3Z%dP37fJpvW494t%d6bxN z#FfTnaTdQ?Cr;)hI6CU3j$3XiT&>u*8)yp&Pxcant63%atFfc^gfxX>dJtpd!P$xd zT8*I}ry15(+2^ZLZ?z>Jrg8L0|7b9jDyjS-`~OkQNV3>6gY2S9f+2;@{82+_vA-28cYd3D|7>-=P`Iir&A(^{*Z+Jzu`=~on)#tl z9)GjcwY#t}f1{^;U%arsai^a8zPZ}=`tsgA$H;{v2U_mlf4V~7>pA&J82WYd^vdMJ qw?9A7@5IHf+h;nybT);S4=(Qybu`x3&0Tx-@?t~d*HCkLLw^C^7q3D9 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0 b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0 new file mode 100644 index 0000000000000000000000000000000000000000..0b805178417800b92a9231d73b46133bb45ea433 GIT binary patch literal 1006 zcmbVLNlwEs6a~Z<78T_HUROClccw8SA_Hm$5LINkiQ5Kcs2mrmRDlz~jum%-gKz^* zfn7=ynhHvHar_3q_id|xv%7yxZds<}!H6d0<8TJ}bWA}TFdAem1TBz~q_)n-B#=Kd zHdKiC9#LgXq9jX|x!91Vlo3uPeGt(&%t5<1`&Pmp>e`xRip_;rlyXqCO2Db6BWywg zm5`jXC>_>Owsu`X#n}qRbPWtu6{%Qi+NIuFq>6OpY=FIhT}N|71Z+FdDOLX{592y< zO+AKYVh`$W^|<{hr}>Zc2E5t;WxBeKJC@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z1{X$dmB;?QQMp0AYY)cson>0ZsIgW5FM~%a6GKvqfYgEI%>V!Iw-hukd!Kh^f7In~ zUfYk}+OKfggK^vY-}`Fi?S) Date: Tue, 4 Aug 2026 18:29:30 +0800 Subject: [PATCH 3/6] fix(parquet): reuse RowGroupPageIndexReader in FileReaderWrapper layer (#166) --- .../format/parquet/column_index_filter.cpp | 12 +--- .../format/parquet/column_index_filter.h | 8 +-- .../parquet/column_index_filter_test.cpp | 49 ++++++++-------- .../format/parquet/file_reader_wrapper.cpp | 41 +++++++++---- .../format/parquet/file_reader_wrapper.h | 9 +++ .../page_filtered_row_group_reader.cpp | 24 ++------ .../parquet/page_filtered_row_group_reader.h | 5 +- .../page_filtered_row_group_reader_test.cpp | 58 ++++++++++++++----- .../parquet/parquet_file_batch_reader.cpp | 8 +-- .../parquet/parquet_file_batch_reader.h | 7 +-- 10 files changed, 129 insertions(+), 92 deletions(-) diff --git a/src/paimon/format/parquet/column_index_filter.cpp b/src/paimon/format/parquet/column_index_filter.cpp index 61849802..1e0a8f59 100644 --- a/src/paimon/format/parquet/column_index_filter.cpp +++ b/src/paimon/format/parquet/column_index_filter.cpp @@ -38,15 +38,9 @@ namespace paimon::parquet { Result ColumnIndexFilter::CalculateRowRanges( const std::shared_ptr& predicate, - const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, - const std::map& column_name_to_index, int32_t row_group_index, - int64_t row_group_row_count) { - if (!predicate || !page_index_reader) { - return RowRanges::CreateSingle(row_group_row_count); - } - - auto rg_page_index_reader = page_index_reader->RowGroup(row_group_index); - if (!rg_page_index_reader) { + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, + const std::map& column_name_to_index, int64_t row_group_row_count) { + if (!predicate || !rg_page_index_reader) { return RowRanges::CreateSingle(row_group_row_count); } diff --git a/src/paimon/format/parquet/column_index_filter.h b/src/paimon/format/parquet/column_index_filter.h index 56bb816e..a3016241 100644 --- a/src/paimon/format/parquet/column_index_filter.h +++ b/src/paimon/format/parquet/column_index_filter.h @@ -57,16 +57,14 @@ class ColumnIndexFilter { /// Calculate row ranges based on predicate and column indices. /// @param predicate The predicate to evaluate. - /// @param page_index_reader The page index reader for the file. + /// @param rg_page_index_reader The page index reader of target row group for the file. /// @param column_name_to_index Map from column name to column index. - /// @param row_group_index The row group index to filter. /// @param row_group_row_count The number of rows in the row group. /// @return RowRanges that may contain matching rows. static Result CalculateRowRanges( const std::shared_ptr& predicate, - const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, - const std::map& column_name_to_index, int32_t row_group_index, - int64_t row_group_row_count); + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, + const std::map& column_name_to_index, int64_t row_group_row_count); private: /// Visit a predicate and calculate row ranges. diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp index 35e02576..94c65ef5 100644 --- a/src/paimon/format/parquet/column_index_filter_test.cpp +++ b/src/paimon/format/parquet/column_index_filter_test.cpp @@ -305,9 +305,8 @@ class ColumnIndexFilterTest : public ::testing::Test { } Result Filter(const std::shared_ptr& predicate) { - return ColumnIndexFilter::CalculateRowRanges(predicate, page_index_reader_, - column_name_to_index_, /*row_group_index=*/0, - row_group_row_count_); + return ColumnIndexFilter::CalculateRowRanges(predicate, page_index_reader_->RowGroup(0), + column_name_to_index_, row_group_row_count_); } std::shared_ptr arrow_pool_; @@ -553,19 +552,19 @@ TEST_F(ColumnIndexFilterTest, SignedZeroUsesJavaOrderForFloatingPointPages) { auto less_negative_zero = PredicateBuilder::LessThan( /*field_index=*/0, /*field_name=*/"value", field_type, field_type == FieldType::FLOAT ? Literal(-0.0f) : Literal(-0.0)); - ASSERT_OK_AND_ASSIGN( - auto ranges, ColumnIndexFilter::CalculateRowRanges( - less_negative_zero, page_index_reader, {{"value", 0}}, - /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_OK_AND_ASSIGN(auto ranges, + ColumnIndexFilter::CalculateRowRanges( + less_negative_zero, page_index_reader->RowGroup(0), {{"value", 0}}, + reader->metadata()->RowGroup(0)->num_rows())); ASSERT_TRUE(ranges.IsEmpty()) << "field type: " << static_cast(field_type); auto less_positive_zero = PredicateBuilder::LessThan( /*field_index=*/0, /*field_name=*/"value", field_type, field_type == FieldType::FLOAT ? Literal(0.0f) : Literal(0.0)); - ASSERT_OK_AND_ASSIGN( - ranges, ColumnIndexFilter::CalculateRowRanges( - less_positive_zero, page_index_reader, {{"value", 0}}, - /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_OK_AND_ASSIGN(ranges, + ColumnIndexFilter::CalculateRowRanges( + less_positive_zero, page_index_reader->RowGroup(0), {{"value", 0}}, + reader->metadata()->RowGroup(0)->num_rows())); ASSERT_EQ(20, ranges.RowCount()); ASSERT_EQ(1, ranges.GetRanges().size()); ASSERT_EQ(0, ranges.GetRanges()[0].from); @@ -576,8 +575,8 @@ TEST_F(ColumnIndexFilterTest, SignedZeroUsesJavaOrderForFloatingPointPages) { field_type == FieldType::FLOAT ? Literal(-0.0f) : Literal(-0.0)); ASSERT_OK_AND_ASSIGN( ranges, ColumnIndexFilter::CalculateRowRanges( - greater_negative_zero, page_index_reader, {{"value", 0}}, - /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + greater_negative_zero, page_index_reader->RowGroup(0), {{"value", 0}}, + reader->metadata()->RowGroup(0)->num_rows())); ASSERT_EQ(30, ranges.RowCount()); auto not_equal_negative_zero = PredicateBuilder::NotEqual( @@ -585,17 +584,17 @@ TEST_F(ColumnIndexFilterTest, SignedZeroUsesJavaOrderForFloatingPointPages) { field_type == FieldType::FLOAT ? Literal(-0.0f) : Literal(-0.0)); ASSERT_OK_AND_ASSIGN( ranges, ColumnIndexFilter::CalculateRowRanges( - not_equal_negative_zero, page_index_reader, {{"value", 0}}, - /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + not_equal_negative_zero, page_index_reader->RowGroup(0), {{"value", 0}}, + reader->metadata()->RowGroup(0)->num_rows())); ASSERT_EQ(30, ranges.RowCount()); auto greater_finite = PredicateBuilder::GreaterThan( /*field_index=*/0, /*field_name=*/"value", field_type, field_type == FieldType::FLOAT ? Literal(2.0f) : Literal(2.0)); - ASSERT_OK_AND_ASSIGN( - ranges, ColumnIndexFilter::CalculateRowRanges( - greater_finite, page_index_reader, {{"value", 0}}, - /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_OK_AND_ASSIGN(ranges, + ColumnIndexFilter::CalculateRowRanges( + greater_finite, page_index_reader->RowGroup(0), {{"value", 0}}, + reader->metadata()->RowGroup(0)->num_rows())); ASSERT_TRUE(ranges.IsEmpty()); auto greater_between_pages = PredicateBuilder::GreaterThan( @@ -603,8 +602,8 @@ TEST_F(ColumnIndexFilterTest, SignedZeroUsesJavaOrderForFloatingPointPages) { field_type == FieldType::FLOAT ? Literal(0.5f) : Literal(0.5)); ASSERT_OK_AND_ASSIGN( ranges, ColumnIndexFilter::CalculateRowRanges( - greater_between_pages, page_index_reader, {{"value", 0}}, - /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + greater_between_pages, page_index_reader->RowGroup(0), {{"value", 0}}, + reader->metadata()->RowGroup(0)->num_rows())); ASSERT_EQ(10, ranges.RowCount()); ASSERT_EQ(1, ranges.GetRanges().size()); ASSERT_EQ(20, ranges.GetRanges()[0].from); @@ -613,10 +612,10 @@ TEST_F(ColumnIndexFilterTest, SignedZeroUsesJavaOrderForFloatingPointPages) { auto equal_finite = PredicateBuilder::Equal( /*field_index=*/0, /*field_name=*/"value", field_type, field_type == FieldType::FLOAT ? Literal(2.0f) : Literal(2.0)); - ASSERT_OK_AND_ASSIGN( - ranges, ColumnIndexFilter::CalculateRowRanges( - equal_finite, page_index_reader, {{"value", 0}}, - /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_OK_AND_ASSIGN(ranges, + ColumnIndexFilter::CalculateRowRanges( + equal_finite, page_index_reader->RowGroup(0), {{"value", 0}}, + reader->metadata()->RowGroup(0)->num_rows())); ASSERT_TRUE(ranges.IsEmpty()); } } diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 39664658..4c90b95f 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -232,15 +232,18 @@ Result> FileReaderWrapper::NextPageFiltered( // Construct the per-RG streaming reader on demand. if (!current_page_filtered_reader_) { const auto& target_rg = target_row_groups_[current_row_group_idx_]; + auto row_group_page_index_reader = GetRowGroupPageIndexReader(rg_id); auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( - target_rg, target_column_indices_, file_reader_->parquet_reader()); + target_rg, target_column_indices_, row_group_page_index_reader, + file_reader_->parquet_reader()); bool pre_buffered = !prebuffered_ranges_.empty(); int64_t max_chunksize = batch_size_ > 0 ? batch_size_ : std::numeric_limits::max(); PAIMON_ASSIGN_OR_RAISE( current_page_filtered_reader_, PageFilteredRowGroupReader::ReadFilteredRowGroup( target_rg, target_column_indices_, file_reader_->properties().cache_options(), - pre_buffered, page_ranges, max_chunksize, pool_, file_reader_.get())); + pre_buffered, page_ranges, max_chunksize, row_group_page_index_reader, pool_, + file_reader_.get())); current_filtered_row_ranges_ = target_rg.GetRowRanges(); current_filtered_rg_start_ = all_row_group_ranges_[rg_id].first; filtered_global_offset_ = 0; @@ -298,6 +301,27 @@ Result> FileReaderWrapper::NextFullyMatched( return record_batch; } +std::shared_ptr<::parquet::RowGroupPageIndexReader> FileReaderWrapper::GetRowGroupPageIndexReader( + int32_t row_group_index) { + auto cached = row_group_page_index_readers_.find(row_group_index); + if (cached != row_group_page_index_readers_.end()) { + return cached->second; + } + + std::shared_ptr<::parquet::RowGroupPageIndexReader> row_group_page_index_reader; + auto page_index_reader = GetPageIndexReader(); + if (page_index_reader) { + row_group_page_index_reader = page_index_reader->RowGroup(row_group_index); + } + + // To avoid OOM, limit the number of row group page index readers cached in memory. + constexpr int32_t kMaxRowGroupPageIndexReaders = 1024; + if (row_group_page_index_readers_.size() < kMaxRowGroupPageIndexReaders) { + row_group_page_index_readers_.emplace(row_group_index, row_group_page_index_reader); + } + return row_group_page_index_reader; +} + Result> FileReaderWrapper::Next() { try { if (PAIMON_UNLIKELY(!reader_initialized_)) { @@ -357,8 +381,9 @@ std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges( if (trg.IsPartiallyMatched()) { // Page-filtered RGs: only matching page byte ranges. + auto row_group_page_index_reader = GetRowGroupPageIndexReader(trg.GetRowGroupIndex()); auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( - trg, column_indices, file_reader_->parquet_reader()); + trg, column_indices, row_group_page_index_reader, file_reader_->parquet_reader()); ranges.insert(ranges.end(), std::make_move_iterator(page_ranges.begin()), std::make_move_iterator(page_ranges.end())); } else { @@ -500,13 +525,9 @@ Result FileReaderWrapper::CalculateFilteredRowRanges( return RowRanges::CreateSingle(row_count); } - auto page_index_reader = GetPageIndexReader(); - if (!page_index_reader) { - return RowRanges::CreateSingle(row_count); - } - - return ColumnIndexFilter::CalculateRowRanges( - predicate, page_index_reader, column_name_to_index, row_group_index, row_count); + return ColumnIndexFilter::CalculateRowRanges(predicate, + GetRowGroupPageIndexReader(row_group_index), + column_name_to_index, row_count); } PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::CalculateFilteredRowRanges") } diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h index 5dbd3817..78a07c79 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.h +++ b/src/paimon/format/parquet/file_reader_wrapper.h @@ -143,6 +143,10 @@ class FileReaderWrapper { int32_t row_group_index, const std::shared_ptr& predicate, const std::map& column_name_to_index); + /// Get or create the page index reader for a row group. + std::shared_ptr<::parquet::RowGroupPageIndexReader> GetRowGroupPageIndexReader( + int32_t row_group_index); + private: FileReaderWrapper(std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, const std::vector>& all_row_group_ranges, @@ -196,6 +200,11 @@ class FileReaderWrapper { // Track pre-buffered ranges so we can wait on destruction std::vector<::arrow::io::ReadRange> prebuffered_ranges_; + + // Arrow caches the file-level PageIndexReader, but RowGroup() creates a new reader each time. + // Keep one reader per row group so its page-index buffers are shared by all read stages. + std::map> + row_group_page_index_readers_; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index c8129dbf..b3402d4b 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -373,6 +373,7 @@ Result> PageFilteredRowGroupReader::Re const TargetRowGroup& target_row_group, const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group_page_index_reader, std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::arrow::FileReader* arrow_file_reader) { auto parquet_reader = arrow_file_reader->parquet_reader(); const auto& row_ranges = target_row_group.GetRowRanges(); @@ -388,14 +389,6 @@ Result> PageFilteredRowGroupReader::Re auto rg_metadata = parquet_reader->metadata()->RowGroup(row_group_index); int64_t row_group_row_count = rg_metadata->num_rows(); - // reuse RowGroupPageIndexReader for multiple columns in the same row group to avoid redundant - // metadata reads - std::shared_ptr<::parquet::RowGroupPageIndexReader> rg_page_index_reader; - auto page_index_reader = parquet_reader->GetPageIndexReader(); - if (page_index_reader) { - rg_page_index_reader = page_index_reader->RowGroup(row_group_index); - } - const auto& manifest = arrow_file_reader->manifest(); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::vector field_indices, @@ -407,8 +400,8 @@ Result> PageFilteredRowGroupReader::Re for (int field_idx : field_indices) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr chunked_array, - ReadFilteredField(rg_page_index_reader, row_group_index, field_idx, column_indices, - row_ranges, row_group_row_count, arrow_file_reader)); + ReadFilteredField(row_group_page_index_reader, row_group_index, field_idx, + column_indices, row_ranges, row_group_row_count, arrow_file_reader)); if (chunked_array->length() != expected_rows) { return Status::Invalid( @@ -434,6 +427,7 @@ Result> PageFilteredRowGroupReader::Re std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRanges( const TargetRowGroup& target_row_group, const std::vector& column_indices, + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group_page_index_reader, ::parquet::ParquetFileReader* parquet_reader) { int32_t row_group_index = target_row_group.GetRowGroupIndex(); const auto& row_ranges = target_row_group.GetRowRanges(); @@ -447,12 +441,6 @@ std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRange auto rg_metadata = file_metadata->RowGroup(row_group_index); int64_t row_group_row_count = rg_metadata->num_rows(); - auto page_index_reader = parquet_reader->GetPageIndexReader(); - std::shared_ptr<::parquet::RowGroupPageIndexReader> rg_page_index_reader; - if (page_index_reader) { - rg_page_index_reader = page_index_reader->RowGroup(row_group_index); - } - for (int32_t col_idx : column_indices) { auto col_chunk = rg_metadata->ColumnChunk(col_idx); const int64_t column_chunk_offset = GetColumnChunkOffset(*col_chunk); @@ -460,8 +448,8 @@ std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRange // Try to get OffsetIndex for page-level ranges std::shared_ptr<::parquet::OffsetIndex> offset_index; - if (rg_page_index_reader) { - offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); + if (row_group_page_index_reader) { + offset_index = row_group_page_index_reader->GetOffsetIndex(col_idx); } if (!offset_index) { diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index 98303e77..683bde71 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -51,18 +51,20 @@ class PageFilteredRowGroupReader { /// Read a row group with page-level filtering. /// @param target_row_group Target row group with index and row ranges /// @param column_indices Leaf column indices to read - /// @param pool Memory pool /// @param cache_options Cache options for PreBuffer /// @param pre_buffered If true, assumes PreBuffer was already called externally /// and only waits via WhenBuffered (no redundant PreBuffer). /// @param page_ranges If non-empty, wait via WhenBufferedRanges instead of WhenBuffered /// @param max_chunksize Per-batch row cap for the returned reader. + /// @param row_group_page_index_reader Reusable page-index reader for the target row group + /// @param pool Memory pool /// @param arrow_file_reader The Arrow FileReader for ColumnReader tree creation /// @return A RecordBatchReader streaming the filtered rows. static Result> ReadFilteredRowGroup( const TargetRowGroup& target_row_group, const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group_page_index_reader, std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::arrow::FileReader* arrow_file_reader); /// Compute the byte ranges of pages that overlap with the given RowRanges. @@ -71,6 +73,7 @@ class PageFilteredRowGroupReader { /// Falls back to entire column chunk range if OffsetIndex is unavailable. static std::vector<::arrow::io::ReadRange> ComputePageRanges( const TargetRowGroup& target_row_group, const std::vector& column_indices, + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group_page_index_reader, ::parquet::ParquetFileReader* parquet_reader); private: diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index a00f895d..67b24ac5 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -690,10 +690,13 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesPartialMatch) { // Single page match: rows [50, 59] = page 5 RowRanges row_ranges; row_ranges.Add(RowRanges::Range(50, 59)); - + auto page_index_reader = parquet_reader->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader); + auto rg_page_index_reader = page_index_reader->RowGroup(0); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), - /*column_indices=*/{0}, parquet_reader.get()); + /*column_indices=*/{0}, /*row_group_page_index_reader=*/rg_page_index_reader, + parquet_reader.get()); // Should have exactly 1 range (page 5 of column 0, no dictionary since disabled) ASSERT_EQ(1, ranges.size()); @@ -715,10 +718,13 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesAllMatch) { // All rows match RowRanges row_ranges; row_ranges.Add(RowRanges::Range(0, 99)); - + auto page_index_reader = parquet_reader->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader); + auto rg_page_index_reader = page_index_reader->RowGroup(0); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}, - parquet_reader.get()); + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), + /*column_indices=*/{0}, + /*row_group_page_index_reader=*/rg_page_index_reader, parquet_reader.get()); // 10 pages, all matching ASSERT_EQ(10, ranges.size()); @@ -740,10 +746,13 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesNoMatch) { auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); RowRanges row_ranges; // empty - + auto page_index_reader = parquet_reader->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader); + auto rg_page_index_reader = page_index_reader->RowGroup(0); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}, - parquet_reader.get()); + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), + /*column_indices=*/{0}, + /*row_group_page_index_reader=*/rg_page_index_reader, parquet_reader.get()); ASSERT_EQ(0, ranges.size()); } @@ -763,9 +772,13 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiColumn) { RowRanges row_ranges; row_ranges.Add(RowRanges::Range(50, 59)); + auto page_index_reader = parquet_reader->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader); + auto rg_page_index_reader = page_index_reader->RowGroup(0); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), - {0, 1}, parquet_reader.get()); + /*column_indices=*/{0, 1}, /*row_group_page_index_reader=*/rg_page_index_reader, + parquet_reader.get()); // 1 matching page per column = 2 ranges total ASSERT_EQ(2, ranges.size()); @@ -790,9 +803,13 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) { row_ranges.Add(RowRanges::Range(20, 29)); row_ranges.Add(RowRanges::Range(70, 79)); + auto page_index_reader = parquet_reader->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader); + auto rg_page_index_reader = page_index_reader->RowGroup(0); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}, - parquet_reader.get()); + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), + /*column_indices=*/{0}, + /*row_group_page_index_reader=*/rg_page_index_reader, parquet_reader.get()); // 2 matching pages for 1 column ASSERT_EQ(2, ranges.size()); @@ -975,9 +992,13 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding) RowRanges row_ranges; row_ranges.Add(RowRanges::Range(0, 99)); + auto page_index_reader = parquet_reader->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader); + auto rg_page_index_reader = page_index_reader->RowGroup(0); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, /*ranges=*/row_ranges), - /*column_indices=*/{0}, parquet_reader.get()); + /*column_indices=*/{0}, /*row_group_page_index_reader=*/rg_page_index_reader, + parquet_reader.get()); ASSERT_FALSE(ranges.empty()); @@ -1419,7 +1440,8 @@ TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpReadsEachLeafDiction auto ranges = PageFilteredRowGroupReader::ComputePageRanges( TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/selected_rows), - /*column_indices=*/{0, 1}, parquet_reader.get()); + /*column_indices=*/{0, 1}, /*row_group_page_index_reader=*/row_group_page_index, + parquet_reader.get()); for (int32_t col_idx = 0; col_idx < 2; ++col_idx) { auto column_chunk = row_group->ColumnChunk(col_idx); @@ -1621,11 +1643,16 @@ TEST_F(PageFilteredRowGroupReaderTest, DictionaryEmptySelectionDoesNotReadPages) ASSERT_GE(column_chunk_offset, 0); ASSERT_GT(column_chunk_end, column_chunk_offset); + auto page_index_reader = arrow_file_reader->parquet_reader()->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader); + auto row_group_page_index = page_index_reader->RowGroup(0); + ASSERT_TRUE(row_group_page_index); RowRanges empty_ranges; TargetRowGroup empty_target(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/empty_ranges); auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( - empty_target, /*column_indices=*/{0}, arrow_file_reader->parquet_reader()); + empty_target, /*column_indices=*/{0}, /*row_group_page_index_reader=*/row_group_page_index, + arrow_file_reader->parquet_reader()); ASSERT_TRUE(page_ranges.empty()); tracking_input->ClearReadAtRanges(); @@ -1633,7 +1660,8 @@ TEST_F(PageFilteredRowGroupReaderTest, DictionaryEmptySelectionDoesNotReadPages) std::unique_ptr result_reader, PageFilteredRowGroupReader::ReadFilteredRowGroup( empty_target, /*column_indices=*/{0}, arrow::io::CacheOptions::Defaults(), - /*pre_buffered=*/false, /*page_ranges=*/{}, /*max_chunksize=*/1024, arrow_pool_, + /*pre_buffered=*/false, /*page_ranges=*/{}, /*max_chunksize=*/1024, + /*row_group_page_index_reader=*/row_group_page_index, arrow_pool_, arrow_file_reader.get())); std::shared_ptr batch; ASSERT_TRUE(result_reader->ReadNext(&batch).ok()); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index fcc47c18..ab142b75 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -408,8 +408,7 @@ Result ParquetFileBatchReader::RefineRowRangesByTrimming( TargetRowGroups target_row_groups; target_row_groups.reserve(src_row_groups.size()); for (const auto& row_group : src_row_groups) { - auto filtered = - TrimRowGroupPageRanges(bitmap, row_group, column_indices, page_index_reader); + auto filtered = TrimRowGroupPageRanges(bitmap, row_group, column_indices); if (!filtered.GetRowRanges().IsEmpty()) { target_row_groups.emplace_back(std::move(filtered)); } @@ -419,10 +418,9 @@ Result ParquetFileBatchReader::RefineRowRangesByTrimming( TargetRowGroup ParquetFileBatchReader::TrimRowGroupPageRanges( const RoaringBitmap32& bitmap, const TargetRowGroup& row_group, - const std::vector& column_indices, - const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader) const { + const std::vector& column_indices) const { int32_t row_group_idx = row_group.GetRowGroupIndex(); - auto rg_page_index_reader = page_index_reader->RowGroup(row_group_idx); + auto rg_page_index_reader = reader_->GetRowGroupPageIndexReader(row_group_idx); if (!rg_page_index_reader) { return row_group; } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 0cdecbfb..7e5e9afa 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -217,10 +217,9 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { // Apply page-level bitmap filtering to a single row group across all // requested columns. Intersects the row group's existing ranges with the // per-column page ranges derived from the bitmap. - TargetRowGroup TrimRowGroupPageRanges( - const RoaringBitmap32& bitmap, const TargetRowGroup& row_group, - const std::vector& column_indices, - const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader) const; + TargetRowGroup TrimRowGroupPageRanges(const RoaringBitmap32& bitmap, + const TargetRowGroup& row_group, + const std::vector& column_indices) const; // Apply bitmap filtering to row ranges by coalescing nearby ranges. Result RefineRowRangesByCoalescing( From 573f829225962b92b9ca0609834112825a3815a7 Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:30:23 +0800 Subject: [PATCH 4/6] fix(parquet): allow reading nested list/map columns whose leaf types differ only in representation (#172) --- .../parquet/parquet_file_batch_reader.cpp | 83 +++++++++- .../parquet_file_batch_reader_test.cpp | 150 ++++++++++++++++++ test/inte/write_and_read_inte_test.cpp | 109 +++++++++++++ 3 files changed, 337 insertions(+), 5 deletions(-) diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index ab142b75..c0cd40e1 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -64,6 +64,67 @@ class Predicate; namespace paimon::parquet { +namespace { +// LIST/MAP do not support pruning fields from their nested value types, but physical and +// logical leaf types may still differ (for example, Parquet reports LTZ timestamps as UTC +// while Paimon exposes them in the local timezone). Compare only the nested projection shape +// here so those representation differences are handled by the normal cast path. +bool HasSameNestedProjectionShape(const std::shared_ptr& read_type, + const std::shared_ptr& file_type) { + const bool read_is_nested = ArrowSchemaValidator::IsNestedType(read_type); + const bool file_is_nested = ArrowSchemaValidator::IsNestedType(file_type); + if (!read_is_nested || !file_is_nested) { + if (read_is_nested || file_is_nested) { + return false; + } + // ParquetTimestampConverter explicitly supports timestamp unit and timezone + // conversion after reading. Other atomic type differences remain unsupported here. + if (read_type->id() == arrow::Type::TIMESTAMP && + file_type->id() == arrow::Type::TIMESTAMP) { + const auto& read_timestamp = static_cast(*read_type); + const auto& file_timestamp = static_cast(*file_type); + return read_timestamp.unit() == file_timestamp.unit() || + (file_timestamp.unit() == arrow::TimeUnit::MILLI && + read_timestamp.unit() == arrow::TimeUnit::SECOND); + } + return read_type->Equals(file_type); + } + if (read_type->id() != file_type->id()) { + return false; + } + + switch (file_type->id()) { + case arrow::Type::STRUCT: { + if (read_type->num_fields() != file_type->num_fields()) { + return false; + } + for (int32_t i = 0; i < file_type->num_fields(); ++i) { + const auto& read_child = read_type->field(i); + const auto& file_child = file_type->field(i); + if (read_child->name() != file_child->name() || + !HasSameNestedProjectionShape(read_child->type(), file_child->type())) { + return false; + } + } + return true; + } + case arrow::Type::LIST: { + const auto& read_list = static_cast(*read_type); + const auto& file_list = static_cast(*file_type); + return HasSameNestedProjectionShape(read_list.value_type(), file_list.value_type()); + } + case arrow::Type::MAP: { + const auto& read_map = static_cast(*read_type); + const auto& file_map = static_cast(*file_type); + return HasSameNestedProjectionShape(read_map.key_type(), file_map.key_type()) && + HasSameNestedProjectionShape(read_map.item_type(), file_map.item_type()); + } + default: + return false; + } +} +} // namespace + ParquetFileBatchReader::ParquetFileBatchReader( std::shared_ptr&& input_stream, std::unique_ptr&& reader, const std::map& options, @@ -667,18 +728,30 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptrtype(), leaf_index); } } - } else if (file_type->id() == arrow::Type::LIST || file_type->id() == arrow::Type::MAP) { + } else if (file_type->id() == arrow::Type::LIST) { // Keep behavior aligned with ORC path: list/map inner partial projection // is currently unsupported and should fail-fast. - if (!read_type->Equals(file_type)) { + if (!HasSameNestedProjectionShape(read_type, file_type)) { return Status::Invalid(fmt::format( "Parquet does not support partial projection inside list/map: src {} vs target {}", file_type->ToString(), read_type->ToString())); } - for (int32_t i = 0; i < file_type->num_fields(); i++) { - PAIMON_RETURN_NOT_OK(CollectLeafIndices( - read_type->field(i)->type(), file_type->field(i)->type(), leaf_index, indices)); + const auto& read_list = static_cast(*read_type); + const auto& file_list = static_cast(*file_type); + PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_list.value_type(), file_list.value_type(), + leaf_index, indices)); + } else if (file_type->id() == arrow::Type::MAP) { + if (!HasSameNestedProjectionShape(read_type, file_type)) { + return Status::Invalid(fmt::format( + "Parquet does not support partial projection inside list/map: src {} vs target {}", + file_type->ToString(), read_type->ToString())); } + const auto& read_map = static_cast(*read_type); + const auto& file_map = static_cast(*file_type); + PAIMON_RETURN_NOT_OK( + CollectLeafIndices(read_map.key_type(), file_map.key_type(), leaf_index, indices)); + PAIMON_RETURN_NOT_OK( + CollectLeafIndices(read_map.item_type(), file_map.item_type(), leaf_index, indices)); } else { // Leaf column — collect its index. indices->push_back((*leaf_index)++); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index dd323460..59e5a25f 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -646,6 +646,156 @@ TEST_F(ParquetFileBatchReaderTest, TestReadSchemaWithMapSelectedKeysMetadata) { << "expected: " << expected_array->ToString() << "\nactual: " << result_array->ToString(); } +TEST_F(ParquetFileBatchReaderTest, TestNestedListTimestampTimezoneAndMapFieldName) { + const std::string timezone = "Asia/Shanghai"; + paimon::test::TimezoneGuard timezone_guard(timezone); + + auto write_attrs_type = + std::make_shared(arrow::field("key", arrow::utf8(), /*nullable=*/false), + arrow::field("attrs", arrow::utf8())); + auto write_element_type = arrow::struct_({ + arrow::field("key", arrow::utf8()), + arrow::field("attrs", write_attrs_type), + arrow::field("updated_at", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)), + }); + auto write_schema = arrow::schema( + {arrow::field("annotations", arrow::list(arrow::field("element", write_element_type)))}); + + const std::string data_json = R"([ + [[ ["ann-1", [["source", "model"]], "2026-07-16 12:00:00.000001"] ]], + [[ ["ann-2", [], "2026-07-16 12:00:00.000002"], + ["ann-3", null, null] ]], + [null] + ])"; + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_schema->fields()), data_json) + .ValueOrDie()); + WriteArray(file_path_, write_array, write_schema, /*write_batch_size=*/write_array->length(), + /*enable_dictionary=*/false, /*max_row_group_length=*/write_array->length()); + + auto read_element_type = arrow::struct_({ + arrow::field("key", arrow::utf8()), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::utf8())), + arrow::field("updated_at", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)), + }); + auto read_schema = arrow::schema( + {arrow::field("annotations", arrow::list(arrow::field("element", read_element_type)))}); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), data_json) + .ValueOrDie()); + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, /*batch_size=*/2); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + auto expected_chunked_array = arrow::ChunkedArray::Make({expected_array}).ValueOrDie(); + ASSERT_TRUE(result_array->Equals(expected_chunked_array)) + << "expected: " << expected_chunked_array->ToString() + << "\nactual: " << result_array->ToString(); + + auto projected_element_type = arrow::struct_({ + arrow::field("key", arrow::utf8()), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::utf8())), + }); + auto partial_read_schema = arrow::schema({arrow::field( + "annotations", arrow::list(arrow::field("element", projected_element_type)))}); + auto c_partial_read_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*partial_read_schema, c_partial_read_schema.get()).ok()); + ASSERT_NOK_WITH_MSG( + parquet_batch_reader->SetReadSchema(c_partial_read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + "Parquet does not support partial projection inside list/map"); + + auto mismatched_element_type = arrow::struct_({ + arrow::field("key", arrow::utf8()), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::utf8())), + arrow::field("updated_at", arrow::utf8()), + }); + auto mismatched_read_schema = arrow::schema({arrow::field( + "annotations", arrow::list(arrow::field("element", mismatched_element_type)))}); + auto c_mismatched_read_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*mismatched_read_schema, c_mismatched_read_schema.get()).ok()); + ASSERT_NOK_WITH_MSG( + parquet_batch_reader->SetReadSchema(c_mismatched_read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + "Parquet does not support partial projection inside list/map"); + + auto unsupported_timestamp_element_type = arrow::struct_({ + arrow::field("key", arrow::utf8()), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::utf8())), + arrow::field("updated_at", arrow::timestamp(arrow::TimeUnit::NANO, timezone)), + }); + auto unsupported_timestamp_schema = arrow::schema({arrow::field( + "annotations", arrow::list(arrow::field("element", unsupported_timestamp_element_type)))}); + auto c_unsupported_timestamp_schema = std::make_unique(); + ASSERT_TRUE( + arrow::ExportSchema(*unsupported_timestamp_schema, c_unsupported_timestamp_schema.get()) + .ok()); + ASSERT_NOK_WITH_MSG(parquet_batch_reader->SetReadSchema(c_unsupported_timestamp_schema.get(), + /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + "Parquet does not support partial projection inside list/map"); +} + +TEST_F(ParquetFileBatchReaderTest, TestNestedTimestampSecondReadFromMilliFile) { + const std::string timezone = "Asia/Shanghai"; + paimon::test::TimezoneGuard timezone_guard(timezone); + + // Parquet has no second-precision timestamp, so the writer stores second timestamps as + // milliseconds. Reading them back with a second-precision schema must cast milli to second, + // including for timestamp leaves nested inside list/struct/map. + auto event_type = arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("ts_sec", arrow::timestamp(arrow::TimeUnit::SECOND)), + arrow::field("ts_tz_sec", arrow::timestamp(arrow::TimeUnit::SECOND, timezone)), + }); + auto schema = arrow::schema({ + arrow::field("events", arrow::list(arrow::field("element", event_type))), + arrow::field("marks", arrow::map(arrow::utf8(), arrow::timestamp(arrow::TimeUnit::SECOND))), + }); + + const std::string data_json = R"([ + [[ ["e-1", "2026-07-16 12:00:01", "2026-07-16 12:00:02"] ], + [["begin", "2026-07-16 12:00:03"]]], + [[ ["e-2", "2026-07-16 12:00:04", null], + ["e-3", null, "2026-07-16 12:00:05"] ], []], + [[null], null] + ])"; + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), data_json) + .ValueOrDie()); + WriteArray(file_path_, write_array, schema, /*write_batch_size=*/write_array->length(), + /*enable_dictionary=*/false, /*max_row_group_length=*/write_array->length()); + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, /*batch_size=*/2); + + // The nested second timestamps are physically stored as milliseconds in the file. + ASSERT_OK_AND_ASSIGN(auto c_file_schema, parquet_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOr(nullptr); + ASSERT_TRUE(file_schema); + auto file_event_type = + static_cast(*file_schema->field(0)->type()).value_type(); + ASSERT_EQ(arrow::Type::STRUCT, file_event_type->id()); + ASSERT_EQ(arrow::TimeUnit::MILLI, + static_cast(*file_event_type->field(1)->type()).unit()); + ASSERT_EQ(arrow::TimeUnit::MILLI, + static_cast(*file_event_type->field(2)->type()).unit()); + auto file_mark_type = + static_cast(*file_schema->field(1)->type()).item_type(); + ASSERT_EQ(arrow::TimeUnit::MILLI, + static_cast(*file_mark_type).unit()); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result_array, + paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get())); + auto expected_array = arrow::ChunkedArray::Make({write_array}).ValueOrDie(); + ASSERT_TRUE(result_array->Equals(expected_array)) + << "expected: " << expected_array->ToString() << "\nactual: " << result_array->ToString(); +} + TEST_F(ParquetFileBatchReaderTest, TestGetFileSchemaWithFieldId) { std::string file_name = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table/bucket-0/" diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index a18fe9d2..53920ba5 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -806,6 +806,115 @@ TEST_P(WriteAndReadInteTest, TestPkTimestampType) { ASSERT_TRUE(success); } +/// End-to-end coverage for second-precision timestamps nested inside list/struct/map. +/// Parquet has no second-precision timestamp, so the writer stores those leaves as milli and +/// the reader has to convert milli back to second for every nested leaf. +TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampSecondPrecision) { + auto [file_format, file_system] = GetParam(); + TimezoneGuard timezone_guard("Asia/Shanghai"); + auto timezone = DateTimeUtils::GetLocalTimezoneName(); + auto event_type = arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("ts_sec", arrow::timestamp(arrow::TimeUnit::SECOND)), + arrow::field("ts_ltz_sec", arrow::timestamp(arrow::TimeUnit::SECOND, timezone)), + }); + arrow::FieldVector fields = { + arrow::field("events", arrow::list(arrow::field("element", event_type))), + arrow::field("marks", arrow::map(arrow::utf8(), arrow::timestamp(arrow::TimeUnit::SECOND))), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, + options, /*is_streaming_mode=*/false)); + std::string data = R"([ + [[["e-1", "1970-01-01 00:00:01", "1970-01-01 00:00:02"]], + [["begin", "1970-01-01 00:00:03"]]], + [[["e-2", "1970-01-01 00:00:04", null], ["e-3", null, "1970-01-01 00:00:05"]], []], + [[null], null] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + std::string expected_data = R"([ + [0, [["e-1", "1970-01-01 00:00:01", "1970-01-01 00:00:02"]], + [["begin", "1970-01-01 00:00:03"]]], + [0, [["e-2", "1970-01-01 00:00:04", null], ["e-3", null, "1970-01-01 00:00:05"]], []], + [0, [null], null] + ])"; + ASSERT_OK_AND_ASSIGN( + bool success, helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), data_splits, + expected_data)); + ASSERT_TRUE(success); +} + +/// End-to-end coverage for TIMESTAMP_LTZ(6) nested inside list/struct/map. The Parquet reader +/// reports LTZ leaves as UTC while the read schema carries the local timezone, so read schema and +/// file schema differ only in the timezone of those leaves; the micro precision stays unchanged. +TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampLtzMicroTimezoneOnly) { + auto [file_format, file_system] = GetParam(); + // Pin a non-UTC timezone so the read schema really differs from what the file reports. + TimezoneGuard timezone_guard("Asia/Shanghai"); + auto timezone = DateTimeUtils::GetLocalTimezoneName(); + auto event_type = arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("ts_ltz_micro", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)), + }); + arrow::FieldVector fields = { + arrow::field("events", arrow::list(arrow::field("element", event_type))), + arrow::field("marks", + arrow::map(arrow::utf8(), arrow::timestamp(arrow::TimeUnit::MICRO, timezone))), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type", "false"}}; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, + options, /*is_streaming_mode=*/false)); + std::string data = R"([ + [[["e-1", "2026-07-16 12:00:00.000001"]], [["begin", "2026-07-16 12:00:00.000002"]]], + [[["e-2", null], ["e-3", "2026-07-16 12:00:00.000003"]], []], + [[null], null] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + std::string expected_data = R"([ + [0, [["e-1", "2026-07-16 12:00:00.000001"]], [["begin", "2026-07-16 12:00:00.000002"]]], + [0, [["e-2", null], ["e-3", "2026-07-16 12:00:00.000003"]], []], + [0, [null], null] + ])"; + ASSERT_OK_AND_ASSIGN( + bool success, helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), data_splits, + expected_data)); + ASSERT_TRUE(success); +} + TEST_P(WriteAndReadInteTest, TestPKWithSequenceFieldInPKField) { arrow::FieldVector fields = { arrow::field("p1", arrow::utf8()), From 794d39475677fada444d38f908511c0f4a83d75c Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Tue, 4 Aug 2026 23:29:49 -0700 Subject: [PATCH 5/6] feat: update commit message to version 12 (#179) * feat: update commit message to version 12 * add compatibility test * fix pre-commit --- src/paimon/core/index/global_index_meta.cpp | 39 +++++- src/paimon/core/index/global_index_meta.h | 8 +- .../index/index_file_meta_serializer_test.cpp | 31 +++++ .../index/index_file_meta_v4_deserializer.h | 125 ++++++++++++++++++ .../table/sink/commit_message_serializer.cpp | 8 +- .../core/table/sink/commit_message_test.cpp | 35 ++++- .../pk_btree_source_meta/README | 32 +++++ ...022e32ce-1d75-4b9a-8496-a3311c44bad0-0.orc | Bin 0 -> 694 bytes ...293d6705-bd19-4986-9aa0-3a24443e7e8e-0.orc | Bin 0 -> 722 bytes ...b4949ff6-78f4-4e58-86ad-33eb3e5e6749-0.orc | Bin 0 -> 694 bytes ...b5896b79-1c4d-4359-920a-ce719ea303af-0.orc | Bin 0 -> 713 bytes ...c2e3ff2f-a3e1-48a1-8bb1-f868477c796a-0.orc | Bin 0 -> 722 bytes .../commit_messages/commit_messages-01 | Bin 0 -> 1633 bytes ...dex-05388eac-7c9d-48cf-9053-f68b800ebe9e-0 | Bin 0 -> 112 bytes ...dex-5d1ef2e5-c136-4059-8380-fe5d60344f83-0 | Bin 0 -> 112 bytes ...dex-759d0483-a0e7-417b-8c3d-288f36679770-0 | Bin 0 -> 96 bytes ...est-0216d085-645d-486b-9ebb-e48df26b8aa7-0 | Bin 0 -> 1970 bytes ...est-5ba1f8c6-fcde-45f8-b59b-a948ef3013be-0 | Bin 0 -> 1970 bytes ...est-d347fbd7-d7cc-4e5e-9f4a-efa8d0ce8651-0 | Bin 0 -> 1961 bytes ...est-180aa8f5-0932-462f-8f02-a1000b585552-0 | Bin 0 -> 3083 bytes ...est-e108640e-b3a6-499d-9821-fe36afde58d6-0 | Bin 0 -> 2815 bytes ...est-e108640e-b3a6-499d-9821-fe36afde58d6-1 | Bin 0 -> 3078 bytes ...est-fbd7c824-c952-4f2b-ad6c-0227a823a745-0 | Bin 0 -> 2833 bytes ...est-fbd7c824-c952-4f2b-ad6c-0227a823a745-1 | Bin 0 -> 3184 bytes ...ist-3fe1dd64-b6ac-46dc-b968-1740e05383a2-0 | Bin 0 -> 392 bytes ...ist-3fe1dd64-b6ac-46dc-b968-1740e05383a2-1 | Bin 0 -> 1533 bytes ...ist-3fe1dd64-b6ac-46dc-b968-1740e05383a2-2 | Bin 0 -> 1533 bytes ...ist-3fe1dd64-b6ac-46dc-b968-1740e05383a2-3 | Bin 0 -> 1543 bytes ...ist-6678a969-de46-4cb3-b744-36a17ac35371-0 | Bin 0 -> 1603 bytes ...ist-6678a969-de46-4cb3-b744-36a17ac35371-1 | Bin 0 -> 1529 bytes ...ist-6678a969-de46-4cb3-b744-36a17ac35371-2 | Bin 0 -> 1713 bytes ...ist-6678a969-de46-4cb3-b744-36a17ac35371-3 | Bin 0 -> 1548 bytes ...ist-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-0 | Bin 0 -> 1723 bytes ...ist-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-1 | Bin 0 -> 1554 bytes .../pk_btree_source_meta/schema/schema-0 | 30 +++++ .../pk_btree_source_meta/snapshot/EARLIEST | 1 + .../pk_btree_source_meta/snapshot/LATEST | 1 + .../pk_btree_source_meta/snapshot/snapshot-1 | 17 +++ .../pk_btree_source_meta/snapshot/snapshot-2 | 18 +++ .../pk_btree_source_meta/snapshot/snapshot-3 | 18 +++ .../pk_btree_source_meta/snapshot/snapshot-4 | 18 +++ .../pk_btree_source_meta/snapshot/snapshot-5 | 18 +++ 42 files changed, 390 insertions(+), 9 deletions(-) create mode 100644 src/paimon/core/index/index_file_meta_v4_deserializer.h create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/README create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-022e32ce-1d75-4b9a-8496-a3311c44bad0-0.orc create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-293d6705-bd19-4986-9aa0-3a24443e7e8e-0.orc create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-b4949ff6-78f4-4e58-86ad-33eb3e5e6749-0.orc create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-b5896b79-1c4d-4359-920a-ce719ea303af-0.orc create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-c2e3ff2f-a3e1-48a1-8bb1-f868477c796a-0.orc create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/commit_messages/commit_messages-01 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-05388eac-7c9d-48cf-9053-f68b800ebe9e-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-5d1ef2e5-c136-4059-8380-fe5d60344f83-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-759d0483-a0e7-417b-8c3d-288f36679770-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-0216d085-645d-486b-9ebb-e48df26b8aa7-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-5ba1f8c6-fcde-45f8-b59b-a948ef3013be-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-d347fbd7-d7cc-4e5e-9f4a-efa8d0ce8651-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-180aa8f5-0932-462f-8f02-a1000b585552-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-e108640e-b3a6-499d-9821-fe36afde58d6-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-e108640e-b3a6-499d-9821-fe36afde58d6-1 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-fbd7c824-c952-4f2b-ad6c-0227a823a745-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-fbd7c824-c952-4f2b-ad6c-0227a823a745-1 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-1 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-2 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-3 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-1 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-2 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-3 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-1 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/schema/schema-0 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/EARLIEST create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/LATEST create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-1 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-2 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-3 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-4 create mode 100644 test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-5 diff --git a/src/paimon/core/index/global_index_meta.cpp b/src/paimon/core/index/global_index_meta.cpp index ffe05e30..ab18280b 100644 --- a/src/paimon/core/index/global_index_meta.cpp +++ b/src/paimon/core/index/global_index_meta.cpp @@ -33,11 +33,20 @@ GlobalIndexMeta::GlobalIndexMeta(int64_t _row_range_start, int64_t _row_range_en int32_t _index_field_id, const std::optional>& _extra_field_ids, const std::shared_ptr& _index_meta) + : GlobalIndexMeta(_row_range_start, _row_range_end, _index_field_id, _extra_field_ids, + _index_meta, nullptr) {} + +GlobalIndexMeta::GlobalIndexMeta(int64_t _row_range_start, int64_t _row_range_end, + int32_t _index_field_id, + const std::optional>& _extra_field_ids, + const std::shared_ptr& _index_meta, + const std::shared_ptr& _source_meta) : row_range_start(_row_range_start), row_range_end(_row_range_end), index_field_id(_index_field_id), extra_field_ids(_extra_field_ids), - index_meta(_index_meta) {} + index_meta(_index_meta), + source_meta(_source_meta) {} bool GlobalIndexMeta::operator==(const GlobalIndexMeta& other) const { if (this == &other) { @@ -49,6 +58,12 @@ bool GlobalIndexMeta::operator==(const GlobalIndexMeta& other) const { if (index_meta && other.index_meta && !(*index_meta == *other.index_meta)) { return false; } + if ((source_meta && !other.source_meta) || (!source_meta && other.source_meta)) { + return false; + } + if (source_meta && other.source_meta && !(*source_meta == *other.source_meta)) { + return false; + } return row_range_start == other.row_range_start && row_range_end == other.row_range_end && index_field_id == other.index_field_id && extra_field_ids == other.extra_field_ids; } @@ -61,14 +76,17 @@ std::string GlobalIndexMeta::ToString() const { std::string index_meta_str = index_meta == nullptr ? "null" : std::string(index_meta->data(), index_meta->size()); + std::string source_meta_str = + source_meta == nullptr ? "null" : std::string(source_meta->data(), source_meta->size()); return fmt::format( "{{row_range_start={}, row_range_end={}, index_field_id={}, extra_field_ids={}, " - "index_meta={}}}", - row_range_start, row_range_end, index_field_id, extra_field_ids_str, index_meta_str); + "index_meta={}, source_meta={}}}", + row_range_start, row_range_end, index_field_id, extra_field_ids_str, index_meta_str, + source_meta_str); } BinaryRow GlobalIndexMeta::ToRow(MemoryPool* pool) const { - BinaryRow row(5); + BinaryRow row(6); BinaryRowWriter writer(&row, 32 * 1024, pool); writer.WriteLong(0, row_range_start); writer.WriteLong(1, row_range_end); @@ -83,6 +101,11 @@ BinaryRow GlobalIndexMeta::ToRow(MemoryPool* pool) const { } else { writer.WriteBinary(4, *index_meta); } + if (source_meta == nullptr) { + writer.SetNullAt(5); + } else { + writer.WriteBinary(5, *source_meta); + } writer.Complete(); return row; } @@ -104,8 +127,13 @@ Result GlobalIndexMeta::FromRow(const InternalRow& row) { index_meta = row.GetBinary(4); assert(index_meta); } + std::shared_ptr source_meta; + if (!row.IsNullAt(5)) { + source_meta = row.GetBinary(5); + assert(source_meta); + } return GlobalIndexMeta(row_range_start, row_range_end, index_field_id, extra_field_ids, - index_meta); + index_meta, source_meta); } const std::shared_ptr& GlobalIndexMeta::DataType() { @@ -117,6 +145,7 @@ const std::shared_ptr& GlobalIndexMeta::DataType() { arrow::list(arrow::field("item", arrow::int32(), /*nullable=*/false)), /*nullable=*/true), arrow::field("_INDEX_META", arrow::binary(), /*nullable=*/true), + arrow::field("_SOURCE_META", arrow::binary(), /*nullable=*/true), }); return schema; } diff --git a/src/paimon/core/index/global_index_meta.h b/src/paimon/core/index/global_index_meta.h index a11cb82a..21599369 100644 --- a/src/paimon/core/index/global_index_meta.h +++ b/src/paimon/core/index/global_index_meta.h @@ -31,12 +31,17 @@ namespace paimon { /// Schema for global index. struct GlobalIndexMeta { - static constexpr int32_t NUM_FIELDS = 5; + static constexpr int32_t NUM_FIELDS = 6; GlobalIndexMeta(int64_t _row_range_start, int64_t _row_range_end, int32_t _index_field_id, const std::optional>& _extra_field_ids, const std::shared_ptr& _index_meta); + GlobalIndexMeta(int64_t _row_range_start, int64_t _row_range_end, int32_t _index_field_id, + const std::optional>& _extra_field_ids, + const std::shared_ptr& _index_meta, + const std::shared_ptr& _source_meta); + bool operator==(const GlobalIndexMeta& other) const; std::string ToString() const; @@ -52,6 +57,7 @@ struct GlobalIndexMeta { int32_t index_field_id; std::optional> extra_field_ids; std::shared_ptr index_meta; + std::shared_ptr source_meta; }; } // namespace paimon diff --git a/src/paimon/core/index/index_file_meta_serializer_test.cpp b/src/paimon/core/index/index_file_meta_serializer_test.cpp index 4db2ce3d..ab1c77b8 100644 --- a/src/paimon/core/index/index_file_meta_serializer_test.cpp +++ b/src/paimon/core/index/index_file_meta_serializer_test.cpp @@ -127,6 +127,37 @@ TEST_F(IndexFileMetaSerializerTest, TestToFromRowWithGlobalIndex) { } } +TEST_F(IndexFileMetaSerializerTest, TestToFromRowWithGlobalIndexWithSourceMeta) { + auto index_meta_bytes = std::make_shared("apple", memory_pool_.get()); + auto source_meta_bytes = std::make_shared("banana", memory_pool_.get()); + IndexFileMetaSerializer serializer(memory_pool_); + GlobalIndexMeta global_index_meta( + /*row_range_start=*/10, /*row_range_end=*/50, + /*index_field_id=*/5, /*extra_field_ids=*/std::optional>({0, 1}), + index_meta_bytes, source_meta_bytes); + { + auto expected = + std::make_shared("bitmap", "bitmap_index_file_0", /*file_size=*/10, + /*row_count=*/41, /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, global_index_meta); + ASSERT_OK_AND_ASSIGN(BinaryRow row, serializer.ToRow(expected)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, serializer.FromRow(row)); + ASSERT_EQ(expected->ToString(), actual->ToString()); + ASSERT_EQ(*expected, *actual); + } + { + // test external path + auto expected = std::make_shared( + "bitmap", "bitmap_index_file_0", /*file_size=*/10, + /*row_count=*/41, /*dv_ranges=*/std::nullopt, + /*external_path=*/"FILE:/tmp/external/bitmap_index_file_0", global_index_meta); + ASSERT_OK_AND_ASSIGN(BinaryRow row, serializer.ToRow(expected)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, serializer.FromRow(row)); + ASSERT_EQ(expected->ToString(), actual->ToString()); + ASSERT_EQ(*expected, *actual); + } +} + TEST_F(IndexFileMetaSerializerTest, TestSerialize) { IndexFileMetaSerializer serializer(memory_pool_); auto expected = GetRandomDeletionVectorIndexFile(); diff --git a/src/paimon/core/index/index_file_meta_v4_deserializer.h b/src/paimon/core/index/index_file_meta_v4_deserializer.h new file mode 100644 index 00000000..c7207f64 --- /dev/null +++ b/src/paimon/core/index/index_file_meta_v4_deserializer.h @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/common/data/internal_array.h" +#include "paimon/common/data/internal_row.h" +#include "paimon/common/utils/linked_hash_map.h" +#include "paimon/core/index/deletion_vector_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/index_file_meta_v2_deserializer.h" +#include "paimon/core/utils/object_serializer.h" +#include "paimon/result.h" + +namespace paimon { +class MemoryPool; + +class IndexFileMetaV4Deserializer : public ObjectSerializer> { + public: + static const std::shared_ptr& GlobalIndexMetaDataType() { + static std::shared_ptr schema = arrow::struct_({ + arrow::field("_ROW_RANGE_START", arrow::int64(), /*nullable=*/false), + arrow::field("_ROW_RANGE_END", arrow::int64(), /*nullable=*/false), + arrow::field("_INDEX_FIELD_ID", arrow::int32(), /*nullable=*/false), + arrow::field("_EXTRA_FIELD_IDS", + arrow::list(arrow::field("item", arrow::int32(), /*nullable=*/false)), + /*nullable=*/true), + arrow::field("_INDEX_META", arrow::binary(), /*nullable=*/true), + }); + return schema; + } + + static const std::shared_ptr& DataType() { + static std::shared_ptr schema = arrow::struct_({ + arrow::field("_INDEX_TYPE", arrow::utf8(), false), + arrow::field("_FILE_NAME", arrow::utf8(), false), + arrow::field("_FILE_SIZE", arrow::int64(), false), + arrow::field("_ROW_COUNT", arrow::int64(), false), + arrow::field("_DELETIONS_VECTORS_RANGES", + arrow::list(arrow::field("item", DeletionVectorMeta::DataType(), true)), + true), + arrow::field("_EXTERNAL_PATH", arrow::utf8(), true), + arrow::field("_GLOBAL_INDEX", GlobalIndexMetaDataType(), true), + }); + return schema; + } + + explicit IndexFileMetaV4Deserializer(const std::shared_ptr& pool) + : ObjectSerializer>(DataType(), pool) {} + + Result ToRow(const std::shared_ptr& meta) const override { + assert(false); + return Status::Invalid("IndexFileMetaV4Deserializer to row is not valid"); + } + + Result> FromRow(const InternalRow& row) const override { + auto file_type = row.GetString(0); + auto file_name = row.GetString(1); + auto file_size = row.GetLong(2); + auto row_count = row.GetLong(3); + std::optional> dv_ranges; + if (!row.IsNullAt(4)) { + dv_ranges = IndexFileMetaV2Deserializer::RowArrayDataToDvRanges(row.GetArray(4).get()); + } + std::optional external_path; + if (!row.IsNullAt(5)) { + external_path = row.GetString(5).ToString(); + } + std::optional global_index_meta; + if (!row.IsNullAt(6)) { + std::shared_ptr global_index_meta_row = + row.GetRow(6, GlobalIndexMetaDataType()->num_fields()); + assert(global_index_meta_row); + int64_t row_range_start = global_index_meta_row->GetLong(0); + int64_t row_range_end = global_index_meta_row->GetLong(1); + int32_t index_field_id = global_index_meta_row->GetInt(2); + std::optional> extra_field_ids; + if (!global_index_meta_row->IsNullAt(3)) { + std::shared_ptr array = global_index_meta_row->GetArray(3); + if (!array) { + return Status::Invalid( + "GlobalIndexMeta FromRow failed with nullptr extra field ids"); + } + PAIMON_ASSIGN_OR_RAISE(extra_field_ids, array->ToIntArray()); + } + std::shared_ptr index_meta; + if (!global_index_meta_row->IsNullAt(4)) { + index_meta = global_index_meta_row->GetBinary(4); + assert(index_meta); + } + global_index_meta = GlobalIndexMeta(row_range_start, row_range_end, index_field_id, + extra_field_ids, index_meta); + } + return std::make_shared(file_type.ToString(), file_name.ToString(), + file_size, row_count, dv_ranges, external_path, + global_index_meta); + } +}; + +} // namespace paimon diff --git a/src/paimon/core/table/sink/commit_message_serializer.cpp b/src/paimon/core/table/sink/commit_message_serializer.cpp index b2533a7f..1b0645f4 100644 --- a/src/paimon/core/table/sink/commit_message_serializer.cpp +++ b/src/paimon/core/table/sink/commit_message_serializer.cpp @@ -31,6 +31,7 @@ #include "paimon/core/index/index_file_meta_v1_deserializer.h" #include "paimon/core/index/index_file_meta_v2_deserializer.h" #include "paimon/core/index/index_file_meta_v3_deserializer.h" +#include "paimon/core/index/index_file_meta_v4_deserializer.h" #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_meta_09_serializer.h" #include "paimon/core/io/data_file_meta_10_serializer.h" @@ -45,7 +46,7 @@ namespace paimon { class MemoryPool; -const int32_t CommitMessageSerializer::CURRENT_VERSION = 11; +const int32_t CommitMessageSerializer::CURRENT_VERSION = 12; CommitMessageSerializer::CommitMessageSerializer(const std::shared_ptr& pool) : memory_pool_(pool), @@ -203,6 +204,11 @@ Result> CommitMessageSerializer::Deserialize(int3 DataInputStream* in) { if (version == CURRENT_VERSION) { return Deserialize(version, data_file_serializer_.get(), index_entry_serializer_.get(), in); + } else if (version == 11) { + auto index_entry_v4_deserializer = + std::make_unique(memory_pool_); + return Deserialize(version, data_file_serializer_.get(), index_entry_v4_deserializer.get(), + in); } else if (version == 9 || version == 10) { auto index_entry_v3_deserializer = std::make_unique(memory_pool_); diff --git a/src/paimon/core/table/sink/commit_message_test.cpp b/src/paimon/core/table/sink/commit_message_test.cpp index 6c00bb94..935e45d2 100644 --- a/src/paimon/core/table/sink/commit_message_test.cpp +++ b/src/paimon/core/table/sink/commit_message_test.cpp @@ -65,6 +65,39 @@ TEST(CommitMessageTest, TestCurrentVersion) { ASSERT_EQ(CommitMessageSerializer::CURRENT_VERSION, CommitMessage::CurrentVersion()); } +TEST(CommitMessageTest, TestCompatibleWithVersion12) { + // index file meta: add global index meta source meta + int32_t version = 12; + std::string data_path = paimon::test::GetDataDir() + + "orc/pk_btree_source_meta.db/pk_btree_source_meta/" + "commit_messages/commit_messages-01"; + auto file_system = std::make_shared(); + auto buffer_length = file_system->GetFileStatus(data_path).value()->GetLen(); + + std::vector buffer(buffer_length, 0); + ASSERT_OK_AND_ASSIGN(auto in_stream, file_system->Open(data_path)); + ASSERT_OK(in_stream->Read(reinterpret_cast(buffer.data()), buffer.size())); + ASSERT_OK(in_stream->Close()); + + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ret, + CommitMessage::Deserialize(version, reinterpret_cast(buffer.data()), + buffer.size(), pool)); + auto res_msg = std::dynamic_pointer_cast(ret); + ASSERT_NE(res_msg, nullptr); + + // check source_meta exists + const auto& new_indexes = res_msg->GetCompactIncrement().NewIndexFiles(); + ASSERT_EQ(new_indexes.size(), 1); + const auto& global_index_meta = new_indexes[0]->GetGlobalIndexMeta(); + ASSERT_TRUE(global_index_meta.has_value()); + ASSERT_NE(global_index_meta->source_meta, nullptr); + + // check result + ASSERT_OK_AND_ASSIGN(std::string serialized_bytes, CommitMessage::Serialize(ret, pool)); + ASSERT_EQ(serialized_bytes, std::string(reinterpret_cast(buffer.data()), buffer.size())); +} + TEST(CommitMessageTest, TestCompatibleWithVersion11) { // index file meta: add global index meta int32_t version = 11; @@ -106,8 +139,6 @@ TEST(CommitMessageTest, TestCompatibleWithVersion11) { // check result ASSERT_EQ(res_msgs, expected_msgs); - ASSERT_OK_AND_ASSIGN(std::string serialized_bytes, CommitMessage::SerializeList(ret, pool)); - ASSERT_EQ(serialized_bytes, std::string(reinterpret_cast(buffer.data()), buffer.size())); } TEST(CommitMessageTest, TestCompatibleWithVersion10) { diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/README b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/README new file mode 100644 index 00000000..692b105e --- /dev/null +++ b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/README @@ -0,0 +1,32 @@ +id:int score:int tag:string +primary key: id +no partition key +no bucket key +bucket count: 1 + +file format: orc +manifest format: orc +deletion-vectors.enabled: true +primary-key btree index: score +compaction.force-rewrite-all-files: true + +Msgs: +snapshot-1: APPEND +Add: 1, 10, first +Add: 2, 20, second + +snapshot-2: COMPACT +Build the primary-key btree index for score. + +snapshot-3: APPEND +Add: 3, 30, third +Add: 4, 40, fourth + +snapshot-4: COMPACT +Rebuild the primary-key btree index for score. + +snapshot-5: COMPACT +Force a full compaction and rebuild the primary-key btree index for score. + +commit_messages-01: +Single CommitMessage serialized by Java CommitMessageSerializer version 12 for snapshot-5. \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-022e32ce-1d75-4b9a-8496-a3311c44bad0-0.orc b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-022e32ce-1d75-4b9a-8496-a3311c44bad0-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..767c7eee21c2cdd44809af2dfa98372c3de750b8 GIT binary patch literal 694 zcmeYdau#G@;9?VE;b012&;~MvxtJLk7=(B@n1t9km;_iP*szE*2rx-tQ_mp4Ai;n| zJuY)Z1T-XU0vKF@_9}3(0__FbEY88CB*?{@mRVF>BE(jlnw+1PA|V#Qz|O$n#;5`$ zofs8>3#%h?$sp81!ZCq%xdkR6ihFz|+CG zsi8)}c*3Dun^-nLEhw&KVsK&+5HCo*wZl!|{KBuV)+gPGmu3uAxRIyj&(*z`J5J6c zQEHU~vxV=1BZZkO4&38Z-hIw~pY0~|qFde})0_9{Pj-`HVF*@a=wa+j+hobczyx-C zLKeeTc7~*LX8H!XcMlv0U}l!!>FKbPZa$Zh^Y%*os;?oN+ak86AKABi{*AWXC0<*~ zrrnqjI{Vr?=~9Qt39ZcM+VYegnf&|}x+Om~Y3kmaGV3GLBqp<3COxL*?Gx0)jXp1# zc`x(e%bR~PFIEVZ@IeRFG#&*3en zCT*|?k~B)@F_pW-5MF;v;^DHap0+(xCWyJOlC)y?ety51$+bO$Gp_N+u5;p6|6D(G z)@`-;#+CPS;F|)8rNU}+&00kHY7$w-5m?cUS$_z;$Q$` zCMI46eVH3i^RM3MnmqFzlc9oshFk~ZPlq&x)~%_W`=AySZ)IYbUwB! zhF9^}8>vgo2ftcq-L%-hwC1q&>waBkj{6^{G1cGP&%bHQNxfW2=~RcvQm3ZCg^w*3 zG+b0Z@Nijj$*Z>oW_K5Pg=iMhJ$)5WFo0S8~W$|y^0GGtn8elUN2yIw7)xb3eg zi?53Qy&9&fdgsq-?kE1cCWUU$y=?S)k$vWyTWfp{Z#gw7?}jF?uiN~$; ztc=5ROho3)I{fcROCICl@5d85A58eOL7zANxVh7I`*z-gxqQ~sZ%D|cuDJBm OIGesOGy4ZQO8@}L+2BI} literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-b4949ff6-78f4-4e58-86ad-33eb3e5e6749-0.orc b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-b4949ff6-78f4-4e58-86ad-33eb3e5e6749-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..767c7eee21c2cdd44809af2dfa98372c3de750b8 GIT binary patch literal 694 zcmeYdau#G@;9?VE;b012&;~MvxtJLk7=(B@n1t9km;_iP*szE*2rx-tQ_mp4Ai;n| zJuY)Z1T-XU0vKF@_9}3(0__FbEY88CB*?{@mRVF>BE(jlnw+1PA|V#Qz|O$n#;5`$ zofs8>3#%h?$sp81!ZCq%xdkR6ihFz|+CG zsi8)}c*3Dun^-nLEhw&KVsK&+5HCo*wZl!|{KBuV)+gPGmu3uAxRIyj&(*z`J5J6c zQEHU~vxV=1BZZkO4&38Z-hIw~pY0~|qFde})0_9{Pj-`HVF*@a=wa+j+hobczyx-C zLKeeTc7~*LX8H!XcMlv0U}l!!>FKbPZa$Zh^Y%*os;?oN+ak86AKABi{*AWXC0<*~ zrrnqjI{Vr?=~9Qt39ZcM+VYegnf&|}x+Om~Y3kmaGV3GLBqp<3COxL*?Gx0)jXp1# zc`x(e%bR~PFIEVZ@IeRFG#&*3en zCT*|?k~B)@F_pW-5MF;v;^DHap0+(xCWyJOlC)y?ety51$+bO$Gp_N+u5;p6|6D(G z)@`-;#+CPS;JA1k3eC9R+!hv|L7Jm=z^JD=x7JNlnEOCIlAZ+3Fq zw6Ojyn(bA}JPA!#JGMwE^DwPwE1LNt@cqIAd%fqCM9$eCe#~^my^dl-!&s%-c&St= z7KY$lh8{LI{%E}uBbe?n_cbIrOde<85 z)EL%xX`LH4Nks~J=eR%L7rsPu{r0Y0#_iV;0aZc&vhOGT5>mJQ#pk}7yK2wcEU%c_ zJ)+P2cTWo4pnKWqby3}|8(Sl1wq;HgeHgIB;TXT5Fq42}!Pn(?8*hf}T_|a+)X(R- zO3|v*`}zGAX1C=T+<6Os>^diJ^)K{8cU`H*H=eqe6SnmSzuaQ+KzFuUiogaN-@Mwc zW$N4OIHBG*HF^n5&*Ed B&{+Tg literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-c2e3ff2f-a3e1-48a1-8bb1-f868477c796a-0.orc b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-c2e3ff2f-a3e1-48a1-8bb1-f868477c796a-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..02673d7e3e6c2696de02badc0d730dcdf61fa6b7 GIT binary patch literal 722 zcmeYdau#G@;9?VE;a~}1&;~MvxtJLk7=(B@ScKR(m;^W^M6if62(U@;U=?SOV89{{ zHAfd{z6iv8UZDBx93lb%5+@h~7@UC;@?5Myi-A^)aj+-}aIvOk78REWv6f_H7Ntn& z1Tb(iFdSfJVggePOhC1aC_*A4K-0l`ic^#G^HRY2((+4-N-}_2+?d0{_AoGLY}Nm( z(6g3_A%&TXje|{qL4q4hGfGG>F|)8rNU}+&00kHY7$w-5m?cUS$_z;$Q$` zCMI46eVH3i^RM3MnmqFzlc9oshFk~ZPlq&x)~%_W`=AySZ)IYbUwB! zhF9^}8>vgo2ftcq-L%-hwC1q&>waBkj{6^{G1cGP&%bHQNxfW2=~RcvQm3ZCg^w*3 zG+b0Z@Nijj$*Z>oW_K5Pg=iMhJ$)5WFo0S8~W$|y^0GGtn8elUN2yIw7)xb3eg zi?53Qy&9&fdgsq-?kE1cCWUU$y=?S)k$vWyTWfp{Z#gw7?}jF?uiN~$; ztc=5ROho3)I{fcROCICl@5d85A58eOL7zANxVh7I`*z-gxqQ~sZ%D|cuDJBm OIGesOGy4ZQO8@}L+2BI} literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/commit_messages/commit_messages-01 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/commit_messages/commit_messages-01 new file mode 100644 index 0000000000000000000000000000000000000000..374b9b8276f970bda9574e8e932c3df30bc028c8 GIT binary patch literal 1633 zcmeHHu}&N@5cLHVogyTVLYs&p1x1GQuD!c5Suz!a0wCzzR43E&y1r%Y(J}c)hip zArs#(z)|G$IKBbH@)f4q4?~>!#q!&Vu^-XRd1=4nII)^9mNw?)q&W8ocbV5eZ(O2 zy$e|8i`nn7C$N|6=!1*~0nKRn!%tCdn&JN+?d=Rkr3r PO0UGL-N$w>X}pg|Q~i09 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-05388eac-7c9d-48cf-9053-f68b800ebe9e-0 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-05388eac-7c9d-48cf-9053-f68b800ebe9e-0 new file mode 100644 index 0000000000000000000000000000000000000000..dcf43f6548cb447c05e52352f344a716b0e45e86 GIT binary patch literal 112 zcmZSKVqjokVq{~ua$uT?MFUJTa{ws@2Knh$S`ZNiRfx!xOI~$gDKtV0%I8HB LX9V&+L!1Ht9Nh(( literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-5d1ef2e5-c136-4059-8380-fe5d60344f83-0 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-5d1ef2e5-c136-4059-8380-fe5d60344f83-0 new file mode 100644 index 0000000000000000000000000000000000000000..dcf43f6548cb447c05e52352f344a716b0e45e86 GIT binary patch literal 112 zcmZSKVqjokVq{~ua$uT?MFUJTa{ws@2Knh$S`ZNiRfx!xOI~$gDKtV0%I8HB LX9V&+L!1Ht9Nh(( literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-759d0483-a0e7-417b-8c3d-288f36679770-0 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-759d0483-a0e7-417b-8c3d-288f36679770-0 new file mode 100644 index 0000000000000000000000000000000000000000..83e0054daafb6ed668020822f645a6a8ca7ea14c GIT binary patch literal 96 zcmZSKVqjokVq{~eK#GAu`(#5GM1(;UA|j;x${Q?&Mo2;VylCQ#K)z>)Qvd)2 Clm#pR literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-0216d085-645d-486b-9ebb-e48df26b8aa7-0 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-0216d085-645d-486b-9ebb-e48df26b8aa7-0 new file mode 100644 index 0000000000000000000000000000000000000000..ee5c88f48e48ed1b30e6d3b8f87ca694411a3f8f GIT binary patch literal 1970 zcmb7CX;c$e7=1H=VG^~302&inN|ZIxkOWNXhJsjdVMG)~h$LiCKtYgHZEZoSRb*2r zK@nw%0(B`a?5>Dn6t#jIB50%_6)lR=Dk#!Nfc_~xr=4@=ee>?zZ@KS2FK<^P0B9IP z4@U^#06&ds6aWBYG>$MZ9Hpa7bOapG0TU3w2J($)`j8JZa2%O~;>>?%yF#|bFncVH zSkYsJ`9i%f9kSe8dL&6p0CLZ5OC>E2lAl}p< ziGn8*2L^k1>T zqx`$$U`1rO-AJPjdGHUMsQ)s%YT(N(xf98Y)t?=sFN{lBo*3D!FRz4;qY@n zEW8c~TavIdY<5U%Ku6{K-kg z@6|4@sTLP}J2PfZJ67A|Uslnb^g4KLmWH`6tw%-rB-*4e?qRg@4~jaspMK5D{obKb z{Z#Cm);rs42FrKJH(|_D?iKsl4Jk?vB3-L>dulIcWoCb@tISNyRyyfVx1@78U+>Rp zv%m5xDL(X^ao^mcnANk2ob7|oY(ArAhdI`kU2AM|$#Qs`K5u5~@yv$!gG~+LW*nRR zy{*l~<}fBKB7`3m%s{-)WX>!qx?# znjx)e;2s7}EwH!UZTGytzdz=~m2=+(gzs*~8F$|ls4S1~>4;oqgw>S(e$~OI#DlZ) zIyEVaF>gZ8_RmCut$rmxo{E23q$$*Vp|q}qR}X}ZiObv#0((FJt0JS~ z&&>Ae?aUpLSihjih2NXk9EuU}nRYR0%zN?eV)mS`0Q+Y&K6z3OX|_q( zT)b^6C%Q%0z)IihbE0?kg@w*tFZBj3y+k36LB4v+ESLDwF_im!%e^U)iPVkb6qBQC z7lb0q7w0-HeaTm3tmmkV%DYv$eR(WthKM~snDTTiQMtxxOIWtoLN{+L41|LaV0lZ^ z_@L^SM`kbUs$NF2H*R`?cHApmLe%`io7c8J|gcsIqmAoXI~!1d|&s!aw zW82+(LmUKY*($5v5Xb&I@;6a!UI`UBb^^id`q(ObC&jaEy?V!q12tWhiH`}V()@1< z+6*I-KODw521oRleQK9(x@&qs_8!EoKEid#mfFY7TWHauDeAneP8Rv?X<6)Qaw?}F z|6^^E-}&fC`9Eo#_x*PCrduF;T~KoJv{gF{zM6l}`57pQ*wA#mGV9jdOw zonDUu%R&Ja(2<9igHhq;i7BiKH1394Wik_%Z$hSbpeBd8*S*)&^s|-4T|cpvTb_T? zV{Nvcto(TWPMK6itdL(zEm+WF=v%FR7A^NJr_T8N2lC*JmJOt+;aTrCg=vPyy0wLd zP)~E!RX^joxW^RBz`V$v*A09oXfI(O4Kf?(^~kXz&pZz5*Lty5xulwnA1K6Ds+nKP z|4=`v8J$vp)YU^}e_T|1vZTwi>6+annRUsdQKoxr9im(_YSR2~HU+qj_|2*-4u$Kx9j-7 E0MwOzY5)KL literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-5ba1f8c6-fcde-45f8-b59b-a948ef3013be-0 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-5ba1f8c6-fcde-45f8-b59b-a948ef3013be-0 new file mode 100644 index 0000000000000000000000000000000000000000..fb7a5f19a3f2ca80eca4e7dd616c0d4247e4dd1e GIT binary patch literal 1970 zcmb7CdpOf;9DjeCt)E$$EgGvsmXZ0f*)SoOMTJ%ovQ ziGUNvM2ke^F+(K?Ksf+r^;RmQjqVsQh$Td#aB>(5T}`mq5EUlcir9p)MPh4_4HC-c z*dkn(jXlES@K}ggWD{n~;&8cQ9tUBOsVf8k14MrKfI~PoT32t3l{U&s6IMbbB}7$D zMi?f+R9FBIG-NtqikcWoO%nzJNQk8-!V|+n$s3}WsbgvJFfc|vbkw6g=7C{A#*B{W zzxu{T`A@~MjL<~6$yy!P;9r{INq8!L5*5a^{4TZ0I-Nvdh5*b(d6FlD)I?>+!n@H9 z(dOMiBKjJ>0sZmGjT`jF16)OS7XY*XP>Zhi304B|e{@BIe1L^|SE>_?l~dP5&uDtJ zcP+Y4v|u(|`2fnE(F1w{AWUto!a`y~kWgz8M=Z7$BOx3S8{zUo z*a%N3WFumpEstww7iwp38-lPbH%5n!Mx&pGI-7eZq<*_=Eg%&hl&8~X2&!bzX+?q5 ztf$g4Q}9ym!p||h67XvOe23%heL)T9KV-fUM-;bF3v!3^MSdBk(xt=XK~@R(Rm#~n zNZC&2gNjGIfZP#RUnhfr<=Z6Ga~93!u8OQ!IfhoG4VJm-H#3 zX3Gub}TF?@dApX4a+;GnQY|*~$l!DB&77rP{#k-0Q3yQAwTmL4r?dlh{ zNg{(WZR1>X!96@U<85x1J@@IiZ{HF=H(%Nj9F=|Kec2wv3oahLbMoaP629f!%T}go z{bG9H4LGxyyzqzNlxbY+q*e8KXVM>?YO7XCjR$l`(%h%#__SDEUUO$XXO;e}#35|A zO|(^@!GOyWxy9aAr)p%{KpnP$d+QokuW%c_L*jVSwXrIx%!QKYcsRqONA%j>(QM9z zye~(|>j!^|AN0@lXlYJ4y05EJdU8ccSzBC+F~vX!_&Hq7ocb59qmJgb1<+1n()XGi z63yBg+h4cKjvn)vqcD&r`JEnFcV&s~+ZP&Rk~U#Q6i`{Q-PveQ%T&MuWjGBgAi_p? zElf@Fv)!7G^KK@1M>@|o>-6{HH>h0raRz#o1v*UD&yqW);WRF>QFYVvIf9J@v9VjS>M|yAu&gVs#=HE;6FzWM>W_zPChNpSq?D_Ys=j+l+SH&IjHufBrSXD2s zDLqX^;=`>9!oGb*;A@OX+-wFKiqSWA?xmOE*$Qb%DOvqq{XuCtdD+o zd}DSDc{({?pmbRVGIm~m5P@mmMEfzOQPaVtcDZ>2JGCvPXkhid+t2$`oie5^y_ms_ zP3_a$=8qMCY&g)pvdK8f+tTJ8PNTrf32{7J;Cvn{>&7u8M;Q_)zpra*j!2z@Ylaqn zVy+;i{Zf_h9QXWKDZOWA&F^hqe|$+hwz=lKdo8oKrQJl%nt6h=T9F?a;32o1!8ps6 z)$MIo)NZPixTVs`Rz_>fH>cg0;x|*-g#R+7P(kX1j-T=@r(;$PUTu$EE~mi`7-W#p6>++PqsnrNR2?+D(l*gIck8cvypkqb4Sjj4^;{_5@;SCV#+UeV6g| F{{|D2Z>azP literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-d347fbd7-d7cc-4e5e-9f4a-efa8d0ce8651-0 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-d347fbd7-d7cc-4e5e-9f4a-efa8d0ce8651-0 new file mode 100644 index 0000000000000000000000000000000000000000..e0bea207dcd438aaa14f7faf19d2ab7b3d4a0e81 GIT binary patch literal 1961 zcmb7CeKb^Q7=Q1~n64QzW^BzQvE(CMhWYdn!$jNWWI|cCEYr*+QVCnKHVu7@jEam< zBg#jqREj=ynh1%rk#c5ZiqcA9YdM`1wbx+((mCxt=RVJUpZj}0e$Vgq58xXDK){o6 zFeCtW=qH7M1ptusV2F%|F(eEXGfNzh02K&;HHtSR=%9Es1BM}Q3{3qxnvbGQXVCLt z$ecvb5yo#9i^)@tN)Uiz0Bo*0PMB@=!GJz2AqIt$MJP0kV83aLNW2r_uv|oRCYOOk z(8U~t>BJEt+zkv7;>_hr7;HAjg~OpEbTaiP0l)xp5I*>%juYB<-V`eXl$9>5f=DWe zs+x^3j)bG)1OP!(j0khI#87IwFc3gOEHwq5Sr$s(6vgOImBqur9F16@5re4+30XXS6(O(IKX+ zHf#+54jqGlzLVYveE~EHTLmZwClBZe0Fl-^iX!485Fyir=^~M^5e`?vM3`b07vZua zLdw0X_p$R-FmKo0%lwE@^s`GMeI!hho zf}wgIKg;xX@Nlk&+qstRbv4&VdMkKd!_WsfPuK^fLLtH;QFbR~0mZ`E6*DAn%C zsa|ey_D3)Or3-=slcK5B_6@dP>O%<@P}t_4WxclC1^I>J?Kko>3KLj5?q(zli(LgL zU)nbOEtf^zqD=Ty?%wEG$+q2aGx6pr<7lUrn&v0nyyJEwIm zSLSDSj9~JL0acw{8u(zN@}RPCB|}L0VN+k3a7aijYmoIG1^Nu0aq@7CJo;&p`~G^F z`-43PkMC{rdXUnqd}PDzk6^Z+dbcpmh($Mw#H7;NeF_r_7Q49f(w*^f35KsmN~Pgv zZIeGMlLx9o8-g74_A@_7L($Rq>|OvX)jvDZJBJ zu6J7c*z;4&W!an_+?1YmSQZ7WeUaozmo@6ER4RC%UM&4UOIldYnPTFdfp?9|12l2# zGYQi8$8Bvfp0B)=Fw{}2BCNgm(3N&S`(#AVP=9LYVsSLs3?hNre237qVcU+kzNT>> zV)TTS#B#Ujv=d4*qog&Lq+45GZExxcZm&pZH4QIJ8hdl}hLi8|)DBD5{%4=7)dOK` z9CuSsoOK!ZHQYK{si^O>G>pnBUQv`&IB$?iXOIr?F6gFOCi5qVic!cgbWB%Ka5gE+Eo4W0p;eZ%Aj;XkYiD~q^LY}f$dS*+RsnU=U@2jH0pNd~(Vqle1pR=e=YGRE;cJXhiPYj3W`qa7%D_rr!!SS}M z+s;%Rl=4XCJIn819;q~z^#uH48v9pp?(=|_m7&gcC7BoQrF(ey|9s)mNSRxe-oCEL zhrty-qJk|iGBVwYeYx*{9Af`Lb&HMbKL0vdgMAm=ZgMvFQpuJ}T^nkmfG!}7e&2mp z^4vSoDtG>MjI^~xX%zOnwX7Rj_Zss)y?p$3z)CyM?Ij9N&7A9t4I0W6{&^Rg^CkCK z{ByK_D6U_6CD$H}7t<6mU6PK-E2O5?;ihtEe_}Ne-^B}=QyP10u&R0UFK}^+4{hf1LjYk)H;avTAL<9GA99C^!n~bhrpy81#zWiRmsP(CwYZ(%C zrPIE@vvEnR-zu3dgQf{9KM)^s#9+OXa!}e?vu1^%0j@vGE=2nKo}}T-T>6cT=IR00 o14>)`IVaGZeJc1+XP*YM!<`|%7(+AOYrK4D^e*mBMFLm*3mjD0& literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-180aa8f5-0932-462f-8f02-a1000b585552-0 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-180aa8f5-0932-462f-8f02-a1000b585552-0 new file mode 100644 index 0000000000000000000000000000000000000000..6015b4710d7389af5031065fcba39d39d211c492 GIT binary patch literal 3083 zcmbtU3piBk8vfUsF_vL6Gl?N(RPJPwpA4syDX^uCDnvL64Ybv<&$M{w9weLmi;|it_*o6L`Z}! zEpIM>0oXsJ?J$^ZzIA93qTHf|(V_*AJV9cfAcZZuh+>3@P>74xZT++8chxOo*H39b zs2f zIrVPnFE47$r*A(rwEP9)cp_9$@-n;tm&_eyfQ~Y-h88e*BddZkMDYAP1}YoHx?#}X zEL6sA`rlBlUS8tAqX+`XF#3!@|3hMwYzl2g*EtBafcxA~P#=f@KmgwebIv=^)er*c zY6NBH@leiIFc>x=k*`${5+9V`XOK60RJB#^S=VM~hHMTG4ji9gcYeHV@oFc(8V)b0 zCNp;+B-Sv^MMQXbZ`+8gWrS|G`#61#%Mu${Y2>>`le0h9%oNJGp@~)D9 z0Kjq7CW!apI))(vaj|ng5Qi%wvJi|#;aHx#{}Ig1{r_j?Sk=fi%#x>n%Q#m*1{H({ zo<2drApji3f}~ZFdxUUZOi@k=Yt}5pV}U#f@%9PwSQO7`$UF4SiTTzWLr#ZGAy4j} zlatNn{#gb915!F`6^DM0b6l1Pg$Ud;lt2HCKLvrS#MqDhC9Up>vTG1>|Mjcug3map{$gnMrL_9_v`6xFi)b69J~EmGx&KgE})mz zaeYg)g-N(&mZa{c#L)z6{Ug3&tk^Kd*>qR@O9!`_qkYbu+fPj)+Pd^9@jb@PK-reH zQY_1GggG}I7DsCTb038zw67qrAlg2cRtynW=LVCgmc}mqeT2Y9`OvPavdPMQB}~3X zAHuEA&ONuYCPVt%>rJ}h9_og1({p{(1~Cc>F278+7n;Z#OG2;v92t`RL53V$Gtm6?UWQA6tyY!Di>O%SmD05m`!d88O^N= zf5+N3bLw-O*noKI#JN(bhHRbX>2ZprqC=!faa#1mA)?dO&YqP6@qC*dr^6tT`iSI?P%PHw<-!zL5eoWvr>P7B2xrhRR%}*0XZ0!okhG(-A|}E9G@wBOAqtY&(!ZJ9QEsF zYP&oGn$!HAe_VKEb-{oC`+i(_WXZCcd@Zj#C0?A$70tU7gXmS>PqhZJBDx-qyuRq5?pt!@&X;_fiKZU9Pgj5E zlF`0v^}mn(=Bs_7{DW1yfr#dbA*uUQW2ZfLm+h_Pd)lM4aZpsk*li56>2730LZnNO zpc#3Py#!nCf4~iSpYn7%=HMpD9jx<5n)CK5yAyoHE)RNH%_;hdH>aI=@JQA#Lh+7^ zLTTm^#q}HGq|8!Ex9P(QdoZ+1*@dm!{ z9K^*glTHua!i%mo^6)rV8e<|zlF*QDuypZjzoD#NojjyT?&Q_-LUr%Lfd8qe*%AY&W+LtGU6cv+la&y zeN&WE^X7tuCSoElPJe{$_YSlV4NsQ^=ZEA-T7Qv@tn_v&W0KS{GB(Dr-*b^NW}=wV zi!U3rEBU_sl2_3}jV`e0OpYEpW!Hknasn0Iu zAX8GM{ercl-Un|h77z{x+cjq0dQ`7xnU}c3VHq-qBl1z0Ok@=dgyaWdnn27H2D8o9 GSp1*T_sNU^ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-e108640e-b3a6-499d-9821-fe36afde58d6-0 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-e108640e-b3a6-499d-9821-fe36afde58d6-0 new file mode 100644 index 0000000000000000000000000000000000000000..816e82b4145d6c9167974f0104d9cc7a5fe485d6 GIT binary patch literal 2815 zcmb7EdpMM78-M1VG2WSRm>Hv9A{vL}ka4ESVA!?eFqE>Lm@#N_%&CvUWF^Va(qUFo zI>>2Pr8BP0XGxo)W@<^qN1L#vNT2kv?=@q)c0bzh>!0U&uHSS2?)!Hip2yeUjRF7> zPsPC)4lqHFYD6pmfT{#zsCXDh#v!-`a6kqK;DF_*J%y-%+M^5@#<)T-@@eaN-XSuC zKpgNV0p$P~T{i!ZZsHaQ&~U_f)HiiIjG^<1=6r4f*NkCl8ER!25^83_x3e*`<~eZ9 z?5!Pa&A3)p78W7a);unsVa8Y)6Bj~7wsHUjkN}L?Nd*g3=OTGts%TyWc$7#UB_gB5 z1;oy!yg+J33nq-Be@ftUXn$430XzaMgvmk}5yJCIHCOPzkM!k&0#Wq;jf6#8J{Rdz z3g+Q+O`M1S4?lk+NeAtgvb4Zws|RyNUQ1!?QP59KDv6eCp5L{eEks{?uLNF>Mm2+9K&;?C21E6 z>+hF74-Ow&b&vM%OIrp)#i4#Q!WP3;J=fSlMv0wk*9YyOI@qATR$@8o&$>!|V3L`% zJ+0uBTYF<)8?bAGPPXz>9PJ_1fvN7N8XiW6tGJn#+sk*9yBzW724Wd{A=(t(b#6gj zoyvQJe#4`YHIq>R8{8Ccc8Pe|HE(Vo+rbqVc0EgvvLQO!XU;yFy=Yel8NiQ*b7;5l@pxqE555 z-NomOC1MegsWBLS+VSCA4V?@{r5w_M+(cz@)$=b(lqHNID>vRTG22upKoaiN$~-~R zN_`Y*6JPYKsWG$sOuNcRAeExnzH4P=ks&EVe}4+G(DO~Xr%glVUQ&v>NrAE7gyZ&% z8{~pf1)jN^g)oj*fFUF*_Y)+`?FwkYXo;}p-Lvv)s?{bQ- zNAP>6`c9nvXXj>!e)4)ho+D^-1kK5}yau~$vr=4<_E!X5Ogw z@Cdx>qRAgPb?f^zBO^0v-1Jou7a90GynMFu@V{w*tST`}H>w(oV^Y^qASwZ43PB*2 zoGCVT=m~gcf6}jf%O(7^xD!LFesX&qB-n zAOq|HW&Y(chopIerHz{%_On)(QM5W2E5LzD9T;0s#9rTMT7%uBcyqoQ`oqqZHUrPqQ8k(=U%&;X1V7*>$E|k&DV#$R+q7* zT5q#G*uPfBiDouZuU8YxE#;1z1GTRZ6z{$#92a#|t7_jX8uq9edcfcCux&_f z<4*+wXU}!dBJb3>*MzT-niuU`w^ndB;SO6mYGBS*p}#KaH+4?#GA}Chx`Ku>xUM<8 zsG-2P`%TG`rtI)ba- zG{=-0hv3e=GwDe6iipX%et@9Q%!d7VlBIo5TaNXn22n52F4?I&wNC0j{>D+kKoFBN z%e&u83wh8>BXyl8E0dDHYDw2+qwJSWBE=Kf@ zOLXsoL$kv#aUWJ>XC~h-;p|l72<_Iy4&Tz!XGiQio#91TH~ZZ(`&prVmcfoF7=(-oEY{cG GP2+b-N+uZq literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-e108640e-b3a6-499d-9821-fe36afde58d6-1 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-e108640e-b3a6-499d-9821-fe36afde58d6-1 new file mode 100644 index 0000000000000000000000000000000000000000..a1a46f2d66157a5ea2a061d5581e3c81f3084ef1 GIT binary patch literal 3078 zcmbtU3piBk8vfTBGcA*045N|TnB)?+aVZMrHVTzVxi#fBC>lwMt=Sjll9Eekc1I|e z+_LF{kbRm`QAtEklS)x)6h$Nz-}}Aax0b7$y*vPT z97zg+nLr=CRKQ~Z03;a%CgBh%q7((0BMyi#1;y*2czHY)#iI-af$d;~J|`Mbz#MTH z%_7AF-w5afK)1$diW?j`#?+6MYE=9B5Ro14x=1&6&|KVSWweLbSQqoAG1 z14Jru5i^5?0aDL3-7%O`SYxyaB}vnAXw%ZjAf7mgrwGV%D4y|Sj=q_5x`9T&>F)2) z{#_Ll1V|&X2r(9+#3Hk%1c+jPzp12$_C)%9Fw*E?Fi1MY1b-6#Uu)sejhLzRXBuY3 zXJ$MrK9k1?6-t(vFN>Owp!r)MN(-v>FGOjeB!Yy53ZkK_#XuueJWjlbfTV=L&P0mn z&2aChZPdD?wB>_%J!h>vp+O{4tG9m!r17SH>j<~FY<4h^t`v4@NHbFBV ziDMWdP+pJ|0*SebinGBm3WsOh{jXq2?*Bit*ebrnu(dP#*9=MYV^BeO5FD~Qat{Cr zQXq|{>L-J1i&YZ0prth%4+BLI8N?3X83+<*@waO@{rs^`O{zdA^?dm|Ks4m;;ry2frF$$AHhJA8nRP1g6JhhF zvWR5m=-BDHh&=UTb7Z=1Pvv8i6f8#5{|cSzWu}Nf?ljex-=Gp5`nc@rWYmSu#_XlH zE5eN}TR(gFN5@pf_tzNc7nqRp1AQJ^TQ7A=qrN!U{Kg{kStl;MliqrM&3+d~%-TFv zE8EmpDa@5W?^uu@A02gq=Z_!oylTg(dRZ3IFovwKT0f@jXWs-2-15~IHowY%sm#tVa;w>?Ol zI07a5uKM!fe8j4ltDGi5vbNwvy(Y)fy>iAl4itt5i}(K5RCa_{uc$6tq#L|4iw}h)Z{jgf6J(tkuz1t3(cKS#2xLTvLjwiUqgWTA+oRNDSyL8p@->DI{ z74$8a61~a&BCrR0w%u;zMW`&M?xF=0d|0oz#bI=OU*wh*0-L0oz#L(#5q_%*a-{b# zp%Zw#BwrZW2K~dqkLrinIWWKIdNbo_Q!8#*)jHr>jY{1Tx%alI3t2H9AI*7|dHwYb zH-(fjoMa_4aS19Qs_L`@)wKenh(rP#T!@z6KXFtD$kY6~$}$P|HZ2aP1RP)Tp zs!RcM0eZA`*X>GGQK=QDHmeD?ZDZS3%Za3eG9eyF2dUtsr~&>cEd4Z4UKEC4h=u6RPkM{iKKlbD#n(ASi zak2|dl2$McfYyiOcU%?h(u5xjyorwtv-{O^i6yRPcoXkT_uJL1?zn7q>w*pG_2rX= zmw66L@U9s@{1~62?5Ch|KZ#=p7p}BtIrw$W+lCBzGOEY~eui!T8@0i^Z|Yv7^-g|l z_TG59UB8=s{!FNyP`M$_u=6Kg>8*#n=wu&PW{I!ayHd^eeFHo$^8&Z7xoNpZk7-W7 z&d0Ijml~Nvx-Ee&l~E0hs?)b>8+|{&(3l(~4H|y{*jTiUIxMvKsX%+->XWt7PB|X%K%?7(uxw#mQWt+o z*yO@;8TLf5=SA};>XE}NizdObYV)aztL480PAa`Gp3ouh9n(jGn&Bv23B z89P2T#jf?V4vx-h=FnfP*kHfjCp=Y!9`t4@_qn5S7cf2ExSze?@TrUW+go0Lczafl zRdvq$F0Hxc3d3)B+_ z2(IB*D7oI8FnWYf zlcpF9{aUAls;;{x)$ou)0u8*9&tpj`X|IanTtD0t6f? zGv&*=WcD_>b*SpKGO4mBN!QAE6dU9S2N`uRvivj80_hIm9mvN`g3F%ILLpL kYuvgab7XlJ6kOx$VftmmfBw*M?g|HL_SyE8upn-%%Q3w=S1~_0?#0+j2 ztRNt(f;$>iM!|7kDpVU~R6x)oic(wwC7^U(6UNgH*fYL&&VB#8|NZX$?(+Wu0uMa^ zvuKIG z6byn11b-l~9{|=#+mG~=RTx0UX{|?n>#v0&MwFIqlqgxm7W15(9mI~#Y;NTIDE54N zo-^Cojw516Mmutyqeb={dr=IVW0Q~+sgDQ)0R|u;5W5=BMR^)(^3p}~BEfPpMNUTK zatMZpGIBn?X zLpMvcbLhXZe!m2Mf7F)L|JlT8_GaGv0wn6(4M^1z%~o^tE&dIF4E2viUsDj1kdldRx;}woSA{KksAUv7478hkIh!?)mrRq z%`Ef}lY0_K-57UK7@{3VBI7sj%~l5XY+b;6*|9GBP-k>+d!guVb4gEh)}dpqyLh4% zAwL+nt7L)m8ip#f6~(Q}nt_PbuU&3Z$4;zz99B^g-7kcu^m%=Ju1W138VX1)&0=OoAtvXQNG>U{on6* z-z5-*VmaoKbM)xJ9^sw$b#kOZgF|tR@8UyNco5HbZWYTK-~L$sy{NRf^T~!ue1hUo zOIAmhWMU7|Fy+n&T^E~XLN;r)XVUB!LA-xfoaivQgW25 zlNS&kdaxwP_lWu0s-qLg*a$`%kZ@39buakW#ALITs*!?Ji+{-S2An;g<@DjbS+wU@ zn9w4p9E$|G&kmyH4$NV{S{k!2|NLxEns_wH_tBlq60efos*$1O&P988>*LDnOwK<) zvt>7)$fUdRuM~~@Q(ZmRXD(ddYBsFrR`b<6!dTJ)fcWM=}9vUw>Meo8WCHez0o|Ei{m=CIVQA)lF`3ycyC6Ucyaj0aio@@nXzqz zg~s5nb(tq6K9nf)1Tr5z7yto|j&ZXh0~iARWj%{Zz0>Sp?Mz?!9!tHI@3YFvN^aa6)&RU-vcq)&dNO%x)z&zEaXwX>yjI_aN!Tn%#;AS?LYW3_}*5tyOKBb0EGxTvET< z@pNp`cG-J!n)8jHgvyo4XBLJ8H1`iRe3u-ic-fX&A9VAAQW85nv`?0|DE^#H*%MPG z>&1uSw|cl<;TX(z&ZrM-PVP3%*)%@7t9H@_qdPi69lZysxPA}zE~D8BPgl9BZ0+9m z8LSVzba(T;!b2wpkFXbSyu5C*uze4b+v7BkvOwNEl=#7=2J+8t>-`Ys5PCLxLvi!a zvBz1Af}O3sl}viVy~xyot>OVjYv?JPug+~law|v$hlQ^K+PX(+6^35l#@^!eh`VwH z?(v=W_f1|8AAFcftBvXS`wl@im-X>yX}Lp1m6a&UqH&C=JL#MGrmRJqY`L`5$vyXu zY5#yvYv0y~3Mbzt#?D+?V>R&?Q_swzs-1^vjVf!zR>-rk6&|O|A}fy-RmG{S_m1_Y zV4BmD?^i#!Gw_KZ%2`{4n{vM`=*WtU>6ezbU%Ps#ZOa~kQ(=Xmpg(8i_@nXrkZ8Z2 zyBPN5+q(w|?rRf^YPoj9`q%Bbn;RIjFara$Sw}D&Qtk*A8zAs7`VSo`GMoSa literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-fbd7c824-c952-4f2b-ad6c-0227a823a745-1 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-fbd7c824-c952-4f2b-ad6c-0227a823a745-1 new file mode 100644 index 0000000000000000000000000000000000000000..7cfbf591f233884d2429c035eec0a546ae1596dd GIT binary patch literal 3184 zcmb_b2{=^y8~)9i#hI}*!=xE4#(FIg#Z$-`qs4JBhKQkvEPw+dM8VjBUl4y10gLz}5ipE4hG2^Yy#WO*fI~ekDLM7E8M7HX+`*UVZ%iYPi!(y_YT zvB$u{GIiU#_`Q!UE^5}Lv^Y_Swz(X9zGuB^(jn_lzn+U2vB|ZG%H1k)6c%NmKsiDQCHSEQ7WqUjED=zb6a-&CA<3sq{xJYToH2r1 z;QyZk_%;5Y75KbRf}zIFcN5To0$LCP^UDx`k6%b=7yw7nD+0qq!ss9lMVLqMfB|#V zpWrymPq!a=uk0+F% z7W;x2=5;o;OR!;iehM^Va6m9-%-loKT|iP802_z{INv@fs`=J&g3X||QO`+*3vk5D z@}p*uVF}*00qS(b_2AJrg>&55b^TS%A%|^`Tm6mJ?mzrSYlhhkqXNlBV`xiDo+dI9?urI3g#C^VedjBd?qT(LdSF6Y)| zCWS@NjnX*&JF-iiRO-nJnob)PO8b;wl;%|JtJObtq;$-XI;v7#{NSL!r~g2@w#qes z&X&(|MF$SsKQk=nab$0R(X^c=ER$z}0z-xQ5A6tKlAzgL}SuQPxl>;{RfOoKP&9JB2lf5s9=sM=4o zpdey6?$)wr^%|t00umY5oWQQdG^mOBzQ5rtn%cjVC*{7!S-OM1U2)No4JohtRN7BQ zJa%lIao(3{y(WStLS8hVHtwLw9%LzsRXT*X52Zyc!m#jRx%-^RcGX~C2 zYTVnwTXL!=f$3*_b#vH~EjrWo5q$y5I`M9-Mp{00Rymn-0%U+3@Muq)ZFCefI;>}U zymzG(6n+JpHlWs_nWs9NMZY)>Ve`?~QE2Qqux7VnuU6f0TuqJK(=w?$9T7Sk;jbvH z6E!qw=1me^zDzfui#BFg3(`Jm=AS+rtKR*_xsKN8qSa=oD`QheCwtW?R_6F!Y!K2m zykc5R3*D+?5v`xTCGVsnbqc?9Ez2Qji@MgG1d(fTjSU>0kYl3(+eg2XI8eL!d@jzt zn@x>`r#|RbfB9WIt8;3&>bD&)b0_mgDo%=DiJNq9YdEb?5z_eVQBki%&#NUJdVAmo z&nwSX^o*={`#$LIv5z9YpKt5W6n(zsrAJuZpHW%3uiO8(I<Jrh zVr@y_l%?k-X6e08$&cw0Ok%>i8jGc~aS3MLJ@?--CeN<04d%JXmp1v8+V^KStZivf zsr)tLy$dIwd)}gow83^wipb5hu{_ZfS6H#nDz*Qyp-<%f-TSvkwBt;wCDvNpdD9V$ zIU6w3`XH#7f&**vs!+Go$r}P4`)QTESKA=ZTYc9znK?%4cacb1Dz>TSy;(zL>prSk zMgFlV)59!@vf4DrE3aD?Cqmv~hPq}pH2!ufGcYjGoSvCD^3`|x#KwHDEn6du&NRKc z$<-8^osb(y^)ejGPA_Rv{wmx%q-A+nR7QaM(qvK*lo?^X*`PB+7xd-PZrmqAr2JMU=THXyBbOz)5j(pkNmO*42JCui8W(vvYxY?E;o?n9g%6H`;485MsvW!!WzoG3)ZQWy+a76KwlZy*i)k#RKI I*1?4OFR{q=MgRZ+ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-0 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-0 new file mode 100644 index 0000000000000000000000000000000000000000..bc2b65e536284e0e4572da77c01377712dab9f10 GIT binary patch literal 392 zcmV;30eAjSQbW=L04TLD{a_IS{R#l~5kfFfskQ+yGcz;MRaKP^!6rJ9uHW<5B+M5# zqG45L;P4DnXl=GD6NvhFUp`MyO+*GW05kwJ0Do6X1e899E-JP?Honq~*~Bovz6P5Dp=R2u6fxlSk;MDs-JVvGrISKb8B& z+%{qJX+B@>8*{r~Im_ATk5liu_2=&HyF1kj0vhF$PGTgZ2ha?2m$X3d{B@hh_{(o; z`{C+g_+pZl8&Jw5qg+xfdT0xmG$W8?LrC%v7)eHwDL~N&a4BEn$asZ?>eDGfv0XgrR%c{h>KjE^n3tKBT3n)=YG`0#W@3=4n`E45rfXtpnWAfHVPvSAmTGL4 zn3j@iYLQ~5Yak>M7r?;8AQ8%_yMWaYXq_0uI)0#aoE-nO1^#JE{L{wb5+(sA2_`J! z3<3-i3;_&Co=^vxDhPBT2nca-Fgh`DNCYro7e#eJ07DWdM+1jI1BXNdhceI@eu(2Z zI2eSOK(=6n57c)0h_i#e1ojlfJ&X{0F~xx*ZVV3b3=Aa34p5&TgFYt%LmY$XKOo16 zp#exbFfd3$t%g!y^ABJZgs8_)0^{aC1DJ4N1ShP728OLH3`%W6A`M4#jW?KWw4TIJ z&vk#pr`z5sJ5!G;wYt92+!>Q6Zq@VX%4N459c*`}9E@4fJ-z9B-nNtyiKCwLSeWwG z@icNbe>azpn>U?bpIKeu1Jgn4nhk<1v;U~1ufFRz<#dYX{S!HUftR&E?=>`D60t?{ z-Lt;jhiRwYs;2Cld9rBV*&dNh<3jU3-yNCLRPXz3v+UTM-N4uebz&^TSyqNgUaA@s zmcRVJ|Nh;(ci+C+%5iPciCnjj54Ud4znA~FMW;$BeQUVuCgGE9(WhKvR?M9-Z)eBp zTb(b@xBt3e^+h?+rM22cLF>)(X4~x7m#$>0Hmoq{wCW5K`kPSQ5+BlW!6(ztu31z5 zxZ&qa?}OR1a->{}`6g!sJMLJnv)R(*jC$k2NQDH;4dDsOS|Mwv?K=34aZSN`3)=!7 zhS(EJW^&lLJb1viFWxXNy(CR+hkdb5jO@`AHfX)@BVSCC&%? zQ*sX8UBVN{5X2D3kg|W4&hANXl%8qtH@e&_J6+~U+_fdgZP&kEs9CnAvZ?<`qIwx$ z9ow&~O3##yV=l)#e>imbVuBBsX44vF^Ls%z!WyOu9PnGQh|8j?eV)>T7tEF6CxT22 zr}H%}yCJP&e$dbPuhOYr^$nWBKEL+p|9&7g-#pDx@~el6giD1P?~2_AT+eSf&&Q}% zY30!S)1O^#_pdNj_SYRd0V@e+wzF2aPH;{xvL{?wjJ7dY4xN`@o7=~GD~Fp zkEkUbaNYiw^-jsPz*QcZ}XMe(v!51q2SVM z+nr|j^KabvDnBb_aspSMt#%8?!iGm%wuc3*`+ui?)&B?oZH~@7XLYOHWKQWCxvCBI z*`bB4S{%;ZoBJkqY8f>u3;U=37Vxuvy<)+s=~EZZQgyYS-7~37M@L(sVNqhTT)%&M z+1W)O8V(#}+qYZdyrk^rJM#TUuLf#J2bPFTyd<&K&-X^>oRhZqUfvX9nNw(OCv@yx z>RbNj3u3BLD~i^n3tKBT3n)=YG`0#W@3=4n`E45rfXtpnWAfHVPvSAmTGL4 zn3j@iYLQ~5Yak>M7r?;8AQ8%_yMWaYXq_0uI)0#aoE-nO1^#JE{L{wb5+(sA2_`J! z3<3-i3;_&Co=^vxDhPBT2nca-Fgh`DNCYro7e#eJ07DWdM+1jI1BXNdhceI@eu(2Z zI2eSOK(=6n57c)0h_i#e1ojlfJ&X{0F~xx*ZVV3b3=Aa34p5&TgFYt%LmY$XKOo16 zp#exbFfd3$t%g!y^ABJZgs8_)0^{aC1DJ4N1ShP728OLH3`%W6A`M4#jW?KWw4TIJ z&vk#pr`z5sJ5!G;wYt92+!>Q6Zq@VX%4N459c*`}9E@4fJ-z9B-nNtyiKCwLSeWwG z@icNbe>azpn>U?bpIKeu1Jgn4nhk<1v;U~1ufFRz<#dYX{S!HUftR&E?=>`D60t?{ z-Lt;jhiRwYs;2Cld9rBV*&dNh<3jU3-yNCLRPXz3v+UTM-N4uebz&^TSyqNgUaA@s zmcRVJ|Nh;(ci+C+%5iPciCnjj54Ud4znA~FMW;$BeQUVuCgGE9(WhKvR?M9-Z)eBp zTb(b@xBt3e^+h?+rM22cLF>)(X4~x7m#$>0Hmoq{wCW5K`kPSQ5+BlW!6(ztu31z5 zxZ&qa?}OR1a->{}`6g!sJMLJnv)R(*jC$k2NQDH;4dDsOS|Mwv?K=34aZSN`3)=!7 zhS(EJW^&lLJb1viFWxXNy(CR+hkdb5jO@`AHfX)@BVSCC&%? zQ*sX8UBVN{5X2D3kg|W4&hANXl%8qtH@e&_J6+~U+_fdgZP&kEs9CnAvZ?<`qIwx$ z9ow&~O3##yV=l)#e>imbVuBBsX44vF^Ls%z!WyOu9PnGQh|8j?eV)>T7tEF6CxT22 zr}H%}yCJP&e$dbPuhOYr^$nWBKEL+p|9&7g-#pDx@~el6giD1P?~2_AT+eSf&&Q}% zY30!S)1O^#_pdNj_SYRd0V@e+wzF2aPH;{xvL{?wjJ7dY4xN`@o7=~GD~Fp zkEkUbaNYiw^-jsPz*QcZ}XMe(v!51q2SVM z+nr|j^KabvDnBb_aspSMt#%8?!iGm%wuc3*`+ui?)&B?oZH~@7XLYOHWKQWCxvCBI z*`bB4S{%;ZoBJkqY8f>u3;U=37Vxuvy<)+s=~EZZQgyYS-7~37M@L(sVNqhTT)%&M z+1W)O8V(#}+qYZdyrk^rJM#TUuLf#J2bPFTyd<&K&-X^>oRhZqUfvX9nNw(OCv@yx z>RbNj3u3BLD~i^n3tKBT3n)=YG`0#W@3=4n`E45rfXtpnWAfHVPvSAmTGL4 zn3j@iYLQ~5YbYcU7r?;8AQ8%_yMWaYXq_0uI)0#aoE$v{0zC#2JqB1@!X&^X!Gv2J z$rI{8Qw4!81OXuq4n`*?4v7E;?4qbH2w+I!9AJQ)U=%pPC~*QFHSAztf&B$>5+fwaFvWo)ZVV3b3=AYj5Kx~VgFYt% zLmYzy50K-;&;TJNp;kjFu=xkD3Ie4Z7#Q%7ehepo(es}HOgJ%u(^f(Q!(J8!r8Xgv zhNHR08_YIZPhzO&y1(JmZSRzwsYjJsU0-SLjL8$X>UnhKvfGXhw!2df#;oX`-t;|h zTS|$c@-O*OoKF2~F9 zvi9e_hMR-dY@PXz?RZ;3@~K<5oz#9lJ992*-`O`JTW0Qx?o+)JGi_$R>h7IKZk8Ts zgSs(+;Vdgdke7CzsPRvldA07Imz1a%+&i?2j9LD4Q|=_$@9O;w6wq<*~db~*m@b`Fcu zj#n4OIf@z+g}2Dc&u(O0xs)fEA%r2AL15a(>o${QEx!qTo+v-_<&iv3zM}WfE$o@pGa-E9kG8#-eb-eoh*z0z4Dd6^Jx7sy_Z!^ zf}CuEihj(i^$IQgnf~Wm@b*a7}c!_^uNwW0f zd5-zBoX!4TojNyHh+TJfhz^6y*GO-zy!AtlvHI{e8S1pKF)rrgu^egQDAa~QY>h-1ef+$N@)*ngX z&n{}o^LGgAC=@Iz43^`UR}fmCDb{?@kmWsd)z*BiW5sUiHzak}_ieh_vB*u3xA)?! zFPxK4=O!M}aoHVB;R#v8v=Y;QuH=*r!S>CPL%KhM= ob7`h?Q$(GUBO^y6ivX)cLxX@469bRNF-8Ns9t0B-rF^GlyKnpoJ{%H&J7)W%*U~vnR0Fwj@ z7I6lkIMD4#-cSdcDhPBV2nca-FgY=C7z8k27e#eJ07DWdM+1jI1BXNdhceI@eu(2Z zI2eSOBp9&T&Ol9ZeW0D95IgyR0RpsF-~^+@31E;x;)WgUDX_O7E@H$b4piyJ=+Xp@ zxwxZ@3@pqfMwTHXL;S2&Je&*+evJD1ybKI+jH3S}cz`B5F)jd;3QAD(C0rR81Q-|? z8MqEGfo(p6TLviaz{r4$1lz`N0vLP$8Nh@WBRKITG%(y{Vc5hbB3f`f_fLIjD1Rvz z>yWWf^y zE|!XUY<=k+2ip(5)!!GlX!`o;jA{xGm=0RkY!GCb|3@Qz`(4K!XC|aRn|<=ey^iA6 zpU+ZGKNC||E)SRdyvLXQU>a9$L3{G+CA$TTPj8=e@y?Z#kMb#JhwhYzIX8NpYQ_`H!)T!>h zK8NYdbfv1u-kt519sEyrxVGF#I$-i9Q;Mzl+@ga?$rj6WCt8}gWHcTu6v>gfU@g}8 z^Iq50iA(u!)KqQN+Q`Vqpl?^F*lm2&<_{BxY@U`%>DKEr=gwq(cV31&Jl^nT$AXDn zOad&BpufiSm7js_PO5!?=Ne`MW8;fgSBdi)o;w?|BPL?&Lbmw~KawsMlx=X;`_bNE zli6|f-dT-*2bZc^IJtZA%LsWg{8vrp-?VfJV<o5I7^f44p{b5Y=DPd4h$4p zW@{|_1x}_FtvZQ2E;*Y#J~Y)%T{nzsO-4;zUyvyq4+28O_EVf(cT7f#`oD*U@(B5=N%XkPW1EtDV@df9mFW(fH~xAo`!90R>5_K2oh)t47ajF4 z%kKJICCYY6PgUc|L9NskyEJBoHJY3aN}P9Uxt1>Xw8-#C+k${L7vXtzS>nJoVN(_yhy4b8lum z{a2y0Zr`CCmw-hbO~*1yvkv(^nRADE?!2iB9`@Y+IMJX?zq;O{{pls+!;_U0%h}y! zn2UvVs^=!!H(F-DW%}|UG^n3tKBT3n)=mXuM7r?;8AQ8%_yMWaYXq_0uI)0#aoE(dE1QzK?EYiW^5+(sA2_`J! z3<3-i3;_&Co=^vxDhPBT2nca-Fgh`DNCYro7e#eJ07DWdM+1jI1BXNdhceI@eu(2Z zI2eSOK(=6n57c)0h_i#e1ojlfJ&X{0F~xx*ZVV3b3=Aa34p5&TgFYt%LmY#c5|HD> z&;TSI7#Jj>RzoSU`3JBHLe%3YfpPPn0ZceBf)iFk1H)Dp2BkJ3k%ptW#v9BwT2ErA z=eob)({1mRovBBaT3ugh?u^M3x9WLx<+9t34z{~f4#uqLp5F95Z(B-<#8J!B$aiMvi?~crAs`vf2S$1sBZeVPKIx&{vEGxq#FI9~R z%U}NgzyGiOs>4bV-9kc~kNM7?>Gk#Kke%i|;ZVY6^JG;|<=Od~%x!m8?42|3=7iI? zPQN_A{8xx|DT~#TW3^2aq<(uRoWFIenaN?}^56m6zIem9^pZ5O9rndKF|tQfl)K!txLzntT6Enw`=eZT$Rffm`Sho?U4Z=RE`wK~^^rQ0Pg*qT#?X=Bn1&n2ApmjdtK zUBVN<5Cnv&`*)exZhoW8X5wi4c%tlVnFd1m@06{S z!oQtn|2goq+6=S(kJoV#|jLXU%Atin!E^GL(( zo8Kf$M50cfob};L$hP+{v@+{o=}b$!`Gx29pbP;*OHbCT||GXe%@l-+zszW?aeKn>}@5|N3QB-Z-*-sqfj()QlVn?fve3a#ygj=f8L%l~{q zOjT+{k@}mmf}Z4vJf^hu#&6UO($~f|$n9Bm+U!a9Q*|a0j_=F@tP%|k0!mB_JQ~Ls P4LF;=Ff;oHIZFZnaf;dl literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-2 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-2 new file mode 100644 index 0000000000000000000000000000000000000000..f4f2396ecb279407bba019d115f005a36e29660b GIT binary patch literal 1713 zcmb_bdo+}39De7UF+L2&B&Ot=W<(g*Xk2o?hN8l*+qm0gQfUkA=pt0A*-7qckPN9L z*W5;GOw#DcJ%PxM1^{t14h17U zfR^BQgE$fZfWyE@92!PpQFx387z04#k?t3jICCU8gO=YyD~&cFC3Z;v@EyI8;lz{k z^{tfC9-9n_U@H$jL-16F_Yy2>bFs5&`YY@Vhav{Cg5Dp+@f%=Zl`AJ@>Y&hK*Q{U#R>a{IE$k%{zB~o7j!g&) z;nC~hK!8IY;2#Nzg0eE=B7%g1j4+I}hG0hzus*6MI4ieiL0Z@yiWp3Tu{0P@gSQGO z=`{+50S>~0b=ATDiR%h9H?C<)2|6Os#c~jM&fonI{Tk|PtTji7wQ>F%7sf|be4FXz z;}D>BX%4EKqj4KSLQm!SD!^Wo{ zk>Aw)+H~l4Kl+4;F;&0L<>`sAk*ciN8pR6r#-=^^Mb;B>eF~w+_eYWMM8uTk=z86DW;cdY zg5^!3k^Fc^`$c7q_6wJgiRU&%F&(02Lk+VOoDS$5{4A2&<&eH1U~uBkC1!5Sk=0hd zGe$BaX{;~MsO@L2SBokpviPnN^>&%eL0KgX(6uFGL@K- z*g&>atWbhR-k1yjHN4;;(%4n_(e@7iaict1GJV9vH;s-??P*Q!~z_Jgg|@AD-pb7$3| zlZJTuUy4;FE@mgOi4$Li|ZgV=&_N-6ZI@fZfERI71-)~T=Zanang=SBBLxX%Xh-i zCe+^4W~ja@I%g>PeR)yBqEArP0iSotb7Po2UJP0EOzC@1Du$kX)rtLTXJw+8gUu2y zT&t|I=n5|^n3tKBT3n)=mXuhojJ z=VV}rV~}_N@?%L+3H*Upm|NSO+d->Pco$IPPMRrY|EqP(?eV>DG-^@&A++`dmuP?qnPR{(X zz}=lc#1~$+`Kqw+mq%uUP|_-cNg>QjCr?OFR(v^AiOWlgXBFeh{w9Bxymsx9^xw~- zwELqUSnr&@xs#orLEr9x#~!!MGY&rBVd3X*SY10iY0tSAIXWkEW?M3@>r2`s=&CHr z%D@5%>}yP0`5D}9Cs`SI9_4iiWxkr~!nEl;$FvZE_MG)s>K6vBk4s&+OhM|$8|}4^ z|Fr7H+g9(lTHNRNF+4wK zmxADu#%7Md=LbV2tzJo&yi`1qk;P`&-n{jTLG47(x&xakUp#Do*;eUU5)vpmWbzOCx@rSg*CCGn*u zw{Nc7=NP`D*Xi%o$vr_A&H999#xU%9wfUOX-mkJHGE10la`Nk{yJ#$QShQrTa-!F! zDJCDc6hC@-?*BXYAMW4W>nqn3b$yZNdn)qLpa0A27IkaQQ=0FD?y2tfu8;lI->1EN z1;hPbUzKR}MTK*96$%z*2FvlwD+sO66ze{y$?~3g*RAhjkF6Fb>%7eU+!8ChbhDtV zyP#t4#aCZAC!et0>277K?RZ9V`Qz+ExAw02*C2Ch>2@CPcP(@I`U-eX`5t&v+V+>F v?%Fo)4+p(VbDgI})Hyjaax}9Duu3#E2q-Zz@MxT5G~jId!p!U+TGOlq}Cldodr>A~U>(}cWI7y=HzSVoT) zNzo;}eNo%S50OUoI~R4IXK6hy+?M;b3W&y~N@B1;`66!kL$7D(hJrpBfWuhw8U-(} zN}1e1APE5Mun?^JNt$5`00G585s=#8@&8F|uwbhq4S*XKh=2$s*Ec%WffMHAkLB>e z7!2ytND9Jw({+xQ*ORyF%d4O7xBMm_%Drs6dW&znXW*uF3 zX?^KSROP_(dO+L+cl^Qkfk~A76Lo#4Lxzy++T5T zZZO)aC$Wv&W?)*I|KKw1&9UHkTu;)FORe;Lbi@h6yBYZTi1GTuxJD&Ex9kVf;U9?O zpL_-aT^xYi7~1@b$l}vB;*lCgmIAy`D-L#WcEw^nzH@SRcJ8FdsJ(EOH*m}EHneUa ziCCkGal1aV*{{*hWzN?}4n%oG+-tef|MXRbqKu(;tQ_g_1<9MlCNoZTh|%E@SKVC< zZ)GCEwZaFdFqc{Mj_IwHQN~)@(cI*q_%bJ*W3wL9o7isBZJ}^Y<4g5?R;H?l+PT*2 zPBl$4ylIm+w16u$9jOKfUs;%V9>11sNd02Z=$oIS zWiTJIRv{#Elr)u~(G^l?Dm5>qqhW7joFMK4Z*FD)BFlo6E1z4i9jmYoFw-YWqm}fN ziLpmmz_~~4)H%+em~q(CmY$rM3B{lw*^CzFWrfkK#ir9~yLPBPqGMJIL|=)Bs#p#< zjGp4N%BKp~${9!gF!cY`aK%qhYM*7+$D6i~V$}xcr0&eElx;Jh)P$J?wv)pa1gxnu zms5|VoT_9m2^7bFJ7UJcIl_w^omq9;Z}9KRQySnB32cc}UmoUwO2rNuD`|My*IP%> zrc-2BVFJakXbZ;Z6F8y%Ym3(r>eg8=dM&f%xFV6Ma3F>k{3zQ(xO*_vB3wHsUeNV^ z*xk-jP7)_aT}qzX7ys0Hrq@q6G%7i*v~9e%bJ^ZDR;)liHzl%6tgULUhaGoyP;e z+Vf$(yLr`P?P-=nd?1Y`Kt$1iml2faDX;qMoC6o~OgV^5H zCo6F$^1bqm*aOue7%eKlPl z3q>85iCkgg?Hu#s7%orKWvsJbfcP86>0zCWuk^e1a+j@bpCtLn37vi#_0Oz5Oo^x<$?a literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-1 b/test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-1 new file mode 100644 index 0000000000000000000000000000000000000000..fa3e6d621677d7d31e452c7a86ee94fc717802b6 GIT binary patch literal 1554 zcmeYdau#G@;9?VE;b074&;~MvxtJLk7=(B@7=_q4SOi!kSOORpF)(Os)&Hwd>dM3* zn?dkpL~h^rWJi33%-F*w9CFytoYWu~PT zm*^T=7$hcIq?zg(SQ;DYnwS}-=~|>280jV&8WZiYBiJsn|}bSAW+JIfdMP&$8Z8D4kUpw^q&DtxG;j#Rzd^AQ5FWJHX)IQ zqq)W#mYTf}=htUeS2)1rXNBMY_U&k&n&5!J#%+y&!MxMdTO$b&g!09$Hg`a z>d0gUSvCeHZZ(Yt&9jWg3PuLZiqZ};2mDsVdqvPQ97r&Sj6Ft0FoD|k% zJ{~+hC#ddw#V+pGO(mXwQ%@^qf@~j9uzU+~eLBMK;g`(ZOhw^w-1rG{HD?DI6sK@g{R;=Qcuyv92s7S#dP=}`>-D>*xdP_IuQJzA zJo6}XhLX6iri4qQg7}%7WYHh@eIzpWrUo;BpwP69*KH=rSbqEPXrf%_!y|c~|31EZ zw()W7@lpffve}kQ`A-zY%l=ieR-Ju#RN64+vc3BorY8?G0=hlBUOcv%7j(gdp_ie- z??4c%_Fv^-p*3F^um9WsD^ca}nWLBX>kF41KVdAteC0gF*yH>cgx}2i_sUmM`tkaG zYhL(OsEG1t*vw`NjlHoX{eo)wgQk@~A4~naI*GYjdGehGRpE=y{BL^iRL*6ttd5uF zzAbz9Rl*c@?%5&Rrj~9wo0dKO>w(6mzU4{IiVOTQesPp-Y;IOSxc05ipfOzbqX*wx_V}FT%NFW zVdwI>!tUk&^8X+Fzw!UiF6XJiOXM>(mUq~9Er?f^x!DouSJ?c4?I!Q9SC{&8qAZWH z{K(@zy~*dag|(hS!J^DyIsW;9Wo^PHsyD9asIVz>%W>S>^8VT}>&3-41mCXfyL7W- z(c+E?$1+NPwRk?6cSm{dyr~M$dM{R`0;~^ip)2YWX2Qc6ST5r)mk`tPa;P v*=OHo{qbPwu5F6mI`#_}a5S Date: Wed, 5 Aug 2026 19:50:48 +0800 Subject: [PATCH 6/6] fix(parquet): avoid seeking into the middle of a row group when prefetch and page index filtering are both enabled (#185) * test: add test cases to show problem * fix: avoid seek to the middle of a RowGroup when reading a parquet with prefetch=on and page-inex-filter=on * test: update test cases * fix: partially-matched path do not push next_row_to_read * test: update test comments and enable multi-thread reading * style: update comments * clang-format --- .../prefetch_file_batch_reader_impl.cpp | 25 +++++- .../reader/prefetch_file_batch_reader_impl.h | 5 ++ .../format/parquet/file_reader_wrapper.cpp | 8 ++ .../parquet/file_reader_wrapper_test.cpp | 53 +++++++++++ test/inte/write_and_read_inte_test.cpp | 90 +++++++++++++++++++ 5 files changed, 178 insertions(+), 3 deletions(-) diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp index 3e8737a3..defdd660 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp @@ -427,6 +427,16 @@ Status PrefetchFileBatchReaderImpl::EnsureReaderPosition( return Status::OK(); } +std::optional> PrefetchFileBatchReaderImpl::FindReadRangeContaining( + size_t reader_idx, uint64_t row_id) const { + for (const auto& range : read_ranges_in_group_[reader_idx]) { + if (row_id >= range.first && row_id < range.second) { + return range; + } + } + return std::nullopt; +} + Status PrefetchFileBatchReaderImpl::HandleReadResult( size_t reader_idx, const std::pair& read_range, ReadBatchWithBitmap&& read_batch_with_bitmap) { @@ -454,9 +464,18 @@ Status PrefetchFileBatchReaderImpl::HandleReadResult( if (0 == slice_end) { // fully out of range, data before global_row_ids has been filtered out - readers_pos_[reader_idx]->store(global_row_ids[0]); - ReaderUtils::ReleaseReadBatch(std::move(read_batch)); - return Status::OK(); + // find the read range that contains the first row id and put it into queue in advance. + std::optional> owner_range = + FindReadRangeContaining(reader_idx, global_row_ids[0]); + if (owner_range == std::nullopt) { + readers_pos_[reader_idx]->store(global_row_ids[0]); + ReaderUtils::ReleaseReadBatch(std::move(read_batch)); + return Status::OK(); + } + // Recurses at most once: global_row_ids[0] is within owner_range, so the recursive + // call cannot compute a zero slice end again. + return HandleReadResult(reader_idx, owner_range.value(), + std::move(read_batch_with_bitmap)); } else if (slice_end < c_array->length) { // partially out of range, data before read_range.second has been effectively consumed readers_pos_[reader_idx]->store(read_range.second); diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h index 36673e8b..f0c302e2 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h @@ -134,6 +134,11 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { Status RefreshReadRangesAfterCleanUp(); Result> EofRange() const; std::optional> GetCurrentReadRange(size_t reader_idx) const; + + /// Find the read range assigned to the given reader that contains the given file row id. + /// Returns nullopt when no assigned range contains it. + std::optional> FindReadRangeContaining(size_t reader_idx, + uint64_t row_id) const; Status EnsureReaderPosition(size_t reader_idx, const std::pair& read_range) const; Status HandleReadResult(size_t reader_idx, const std::pair& read_range, diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 4c90b95f..48a4430a 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -259,6 +259,14 @@ Result> FileReaderWrapper::NextPageFiltered( static_cast(*original_row) : current_filtered_rg_start_; filtered_global_offset_ += record_batch->num_rows(); + // Advance to the next row that survives filtering, or to the row group end when the + // filtered ranges are exhausted, so that next_row_to_read_ tracks the streaming position. + auto next_original_row = + current_filtered_row_ranges_.MapFilteredIndexToOriginalRow(filtered_global_offset_); + next_row_to_read_ = + next_original_row.has_value() + ? current_filtered_rg_start_ + static_cast(*next_original_row) + : all_row_group_ranges_[rg_id].second; return record_batch; } diff --git a/src/paimon/format/parquet/file_reader_wrapper_test.cpp b/src/paimon/format/parquet/file_reader_wrapper_test.cpp index 3ac7f62e..de3b6fdc 100644 --- a/src/paimon/format/parquet/file_reader_wrapper_test.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper_test.cpp @@ -374,6 +374,59 @@ TEST_F(FileReaderWrapperTest, PageFilteredRespectsBatchSize) { } } +/// While streaming a page-filtered row group, GetNextRowToRead() must report the next row that +/// survives filtering, and the row group end once the filtered ranges are exhausted. Reporting the +/// row group start for the whole row group makes callers believe the reader has not moved. +TEST_F(FileReaderWrapperTest, PageFilteredAdvancesNextRowToRead) { + std::string file_path = PathUtil::JoinPath(dir_->Str(), "page_next_row.parquet"); + // 2000 rows produces 2 row groups (max_row_group_length=1000) with page index enabled. + PrepareParquetFile(file_path, /*row_count=*/2000, /*enable_page_index=*/true); + ASSERT_OK_AND_ASSIGN(auto reader_wrapper, + PrepareReaderWrapper(file_path, /*wrapper_batch_size=*/7)); + ASSERT_EQ(2, reader_wrapper->GetNumberOfRowGroups()); + + // RowRanges are RG-local. RG0 keeps two non-contiguous stretches so that a batch can span the + // gap between them; RG1 keeps its first 20 rows. + RowRanges rg0_ranges({RowRanges::Range(10, 49), RowRanges::Range(100, 149)}); + RowRanges rg1_ranges(RowRanges::Range(0, 19)); + ASSERT_OK(reader_wrapper->PrepareForReading( + {TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/rg0_ranges), + TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/true, /*ranges=*/rg1_ranges)}, + /*column_indices=*/{0, 1, 2})); + + // Absolute row numbers the reader is expected to produce, in order. + std::vector expected_rows; + for (uint64_t row = 10; row <= 49; ++row) { + expected_rows.push_back(row); + } + for (uint64_t row = 100; row <= 149; ++row) { + expected_rows.push_back(row); + } + for (uint64_t row = 1000; row <= 1019; ++row) { + expected_rows.push_back(row); + } + + size_t consumed = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr record_batch, + reader_wrapper->Next()); + if (!record_batch) { + break; + } + ASSERT_LT(consumed, expected_rows.size()); + ASSERT_EQ(expected_rows[consumed], + reader_wrapper->GetPreviousBatchFirstRowNumber().value()); + consumed += record_batch->num_rows(); + ASSERT_LE(consumed, expected_rows.size()); + // RG0 ends exactly where RG1 starts, so the row group boundary is also covered by + // expected_rows; only the very last batch leaves the cursor at the file end. + uint64_t expected_next_row = + consumed < expected_rows.size() ? expected_rows[consumed] : 2000; + ASSERT_EQ(expected_next_row, reader_wrapper->GetNextRowToRead()); + } + ASSERT_EQ(expected_rows.size(), consumed); +} + TEST_F(FileReaderWrapperTest, GetRowGroupRanges) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); PrepareParquetFile(file_path, /*row_count=*/5500); diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 53920ba5..dd63ed6b 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -1529,6 +1529,96 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } +/// Reproduces the prefetch + parquet page-index filter failure: the predicate keeps only the last +/// page of RG1, RG2 and RG3, so the prefetch reader ends up seeking to a row in the middle of a row +/// group, which FileReaderWrapper::SeekToRow rejects. +TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilterAndPrefetch) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" || file_system != "local") { + return; + } + + auto test_dir = UniqueTestDirectory::Create("local"); + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8())}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, "local"}, + // One row per page (see TestAppendWithParquetPageIndexFilter for why these three + // options are needed together) and 4 rows per row group, so the 16 rows below end up + // in 4 row groups of 4 single-row pages. + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + {"parquet.write.max-row-group-length", "4"}, + {"parquet.read.enable-page-index-filter", "true"}, + }; + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir->Str(), schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); + std::string table_path = test_dir->Str() + "/foo.db/bar"; + + std::string data = R"([ + [0, "v0"], [1, "v1"], [2, "v2"], [3, "v3"], + [4, "v4"], [5, "v5"], [6, "v6"], [7, "v7"], + [8, "v8"], [9, "v9"], [10, "v10"], [11, "v11"], + [12, "v12"], [13, "v13"], [14, "v14"], [15, "v15"] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + // Keep only the last row of RG1, RG2 and RG3, so each row group is partially matched and its + // first selected row is 3 rows behind the row group start. + auto predicate = PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::INT, + {Literal(7), Literal(11), Literal(15)}); + ASSERT_TRUE(predicate); + + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options) + .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_FALSE(result_plan->Splits().empty()); + + // The row group aligned read ranges still cover RG0 although the predicate pruned that row + // group, and 2 sub readers take the ranges round robin. The reader0 owning RG0's range + // therefore finds no data for it and skips ahead into the next row group it owns, whose first + // selected row sits in the middle of that row group. Row level filtering stays off: the + // expected rows below are exactly what page-index filtering selects. + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options) + .SetPredicate(predicate) + .EnablePrefetch(true) + .SetPrefetchMaxParallelNum(2) + .SetPrefetchBatchCount(3) + .AddOption("test.enable-adaptive-prefetch-strategy", "false"); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_data_type = arrow::struct_(fields_with_row_kind); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_data_type, R"([ +[0, 7, "v7"], [0, 11, "v11"], [0, 15, "v15"] +])") + .ValueOrDie()); + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); +} + TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { auto [file_format, file_system] = GetParam(); if (file_format != "parquet" || file_system != "local") {